def copy_package(fd, basedir, relpath, checksum_type, checksum, force=None):
"""
Copies the information from the file descriptor to a file
Checks the file's checksum, raising FileConflictErrror if it's different
The force flag prevents the exception from being raised, and copies the
file even if the checksum has changed
"""
packagePath = basedir + "/" + relpath
# Is the file there already?
if os.path.isfile(packagePath) and not force:
# Get its checksum
localsum = getFileChecksum(checksum_type, packagePath)
if checksum == localsum:
# Same file, so get outa here
return
raise FileConflictError(os.path.basename(packagePath))
dir = os.path.dirname(packagePath)
# Create the directory where the file will reside
if not os.path.exists(dir):
createPath(dir)
pkgfd = os.open(packagePath, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
os.lseek(fd, 0, 0)
while 1:
buffer = os.read(fd, 65536)
if not buffer:
break
n = os.write(pkgfd, buffer)
if n != len(buffer):
# Error writing to the file
raise IOError, "Wrote %s out of %s bytes in file %s" % (
n, len(buffer), packagePath)
os.close(pkgfd)
# set the path perms readable by all users
setPermsPath(packagePath, chmod=0644)
def write_file(self, stream_in):
"""Writes the contents of stream_in to the filesystem
Returns the file size(success) or raises FileCreationError"""
dirname = os.path.dirname(self.full_path)
createPath(dirname)
stat = os.statvfs(dirname)
f_bsize = stat[0] # file system block size
# misa: it's kind of icky whether to use f_bfree (free blocks) or
# f_bavail (free blocks for non-root). f_bavail is more correct, since
# you don't want to have the system out of disk space because of
# satsync; but people would get confused when looking at the output of
# df
f_bavail = stat[4] # free blocks
freespace = f_bsize * float(f_bavail)
if self.file_size is not None and self.file_size > freespace:
msg = messages.not_enough_diskspace % (freespace / 1024)
log(-1, msg, stream=sys.stderr)
# pkilambi: As the metadata download does'nt check for unfetched rpms
# abort the sync when it runs out of disc space
sys.exit(-1)
#raise FileCreationError(msg)
if freespace < 5000 * 1024: # arbitrary
msg = messages.not_enough_diskspace % (freespace / 1024)
log(-1, msg, stream=sys.stderr)
# pkilambi: As the metadata download does'nt check for unfetched rpms
# abort the sync when it runs out of disc space
sys.exit(-1)
#raise FileCreationError(msg)
fout = open(self.full_path, 'wb')
# setting file permissions; NOTE: rhnpush uses apache to write to disk,
# hence the 6 setting.
setPermsPath(self.full_path, user='******', group='apache', chmod=0644)
size = 0
try:
while 1:
buf = stream_in.read(self.buffer_size)
if not buf:
break
buf_len = len(buf)
fout.write(buf)
size = size + buf_len
except IOError, e:
msg = "IOError: %s" % e
log(-1, msg, stream=sys.stderr)
# Try not to leave garbage around
try:
os.unlink(self.full_path)
except (OSError, IOError):
pass
raise FileCreationError(msg), None, sys.exc_info()[2]
def write_file(self, stream_in):
"""Writes the contents of stream_in to the filesystem
Returns the file size(success) or raises FileCreationError"""
dirname = os.path.dirname(self.full_path)
createPath(dirname)
stat = os.statvfs(dirname)
f_bsize = stat[0] # file system block size
# misa: it's kind of icky whether to use f_bfree (free blocks) or
# f_bavail (free blocks for non-root). f_bavail is more correct, since
# you don't want to have the system out of disk space because of
# satsync; but people would get confused when looking at the output of
# df
f_bavail = stat[4] # free blocks
freespace = f_bsize * float(f_bavail)
if self.file_size is not None and self.file_size > freespace:
msg = messages.not_enough_diskspace % (freespace / 1024)
log(-1, msg, stream=sys.stderr)
# pkilambi: As the metadata download does'nt check for unfetched rpms
# abort the sync when it runs out of disc space
sys.exit(-1)
#raise FileCreationError(msg)
if freespace < 5000 * 1024: # arbitrary
msg = messages.not_enough_diskspace % (freespace / 1024)
log(-1, msg, stream=sys.stderr)
# pkilambi: As the metadata download does'nt check for unfetched rpms
# abort the sync when it runs out of disc space
sys.exit(-1)
#raise FileCreationError(msg)
fout = open(self.full_path, 'wb')
# setting file permissions; NOTE: rhnpush uses apache to write to disk,
# hence the 6 setting.
setPermsPath(self.full_path, user='******', group='apache', chmod=0644)
size = 0
try:
while 1:
buf = stream_in.read(self.buffer_size)
if not buf:
break
buf_len = len(buf)
fout.write(buf)
size = size + buf_len
except IOError, e:
msg = "IOError: %s" % e
log(-1, msg, stream=sys.stderr)
# Try not to leave garbage around
try:
os.unlink(self.full_path)
except (OSError, IOError):
pass
raise FileCreationError(msg), None, sys.exc_info()[2]
def _safe_create(fname, user, group, mode):
""" This function returns a file descriptor for the open file fname
If the file is already there, it is truncated
otherwise, all the directories up to it are created and the file is created
as well.
"""
# There can be race conditions between the moment we check for the file
# existence and when we actually create it, so retry if something fails
tries = 5
while tries:
tries = tries - 1
# we're really picky about what can we do
if os.access(fname, os.F_OK): # file exists
if not os.access(fname, os.R_OK | os.W_OK):
raise UnreadableFileError()
fd = os.open(fname, os.O_WRONLY | os.O_TRUNC)
# We're done
return fd
# If directory does not exist, attempt to create it
dirname = os.path.dirname(fname)
if not os.path.isdir(dirname):
try:
#os.makedirs(dirname, 0755)
makedirs(dirname, mode, user, group)
except OSError:
e = sys.exc_info()[1]
# There is a window between the moment we check the disk and
# the one we try to create the directory
# We double-check the file existance here
if not (e.errno == EEXIST and os.path.isdir(dirname)):
# If the exception was thrown on a parent dir
# check the subdirectory to go through next loop.
if os.path.isdir(e.filename):
continue
# Pass exception through
raise
except:
# Pass exception through
raise
# If we got here, it means the directory exists
# file does not exist, attempt to create it
# we pass most of the exceptions through
try:
fd = os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL,
int('0644', 8))
except OSError:
e = sys.exc_info()[1]
# The file may be already there
if e.errno == EEXIST and os.access(fname, os.F_OK):
# Retry
continue
# Pass exception through
raise
# If we got here, the file is created, so break out of the loop
setPermsPath(fname, user, group, mode)
return fd
# Ran out of tries; something is fishy
# (if we manage to create or truncate the file, we've returned from the
# function already)
raise RuntimeError("Attempt to create file %s failed" % fname)
def sync(self, update_repodata=True):
"""Trigger a reposync"""
failed_packages = 0
sync_error = 0
if not self.urls:
sync_error = -1
start_time = datetime.now()
for (repo_id, url, repo_label) in self.urls:
log(0, "Repo URL: %s" % url)
plugin = None
# If the repository uses a uln:// URL, switch to the ULN plugin, overriding the command-line
if url.startswith("uln://"):
self.repo_plugin = self.load_plugin("uln")
# pylint: disable=W0703
try:
if repo_label:
repo_name = repo_label
else:
# use modified relative_url as name of repo plugin, because
# it used as name of cache directory as well
relative_url = '_'.join(url.split('://')[1].split('/')[1:])
repo_name = relative_url.replace("?", "_").replace(
"&", "_").replace("=", "_")
plugin = self.repo_plugin(url,
repo_name,
org=str(self.org_id or ''),
channel_label=self.channel_label)
if update_repodata:
plugin.clear_cache()
if repo_id is not None:
keys = rhnSQL.fetchall_dict("""
select k1.key as ca_cert, k2.key as client_cert, k3.key as client_key
from rhncontentsource cs inner join
rhncontentsourcessl csssl on cs.id = csssl.content_source_id inner join
rhncryptokey k1 on csssl.ssl_ca_cert_id = k1.id left outer join
rhncryptokey k2 on csssl.ssl_client_cert_id = k2.id left outer join
rhncryptokey k3 on csssl.ssl_client_key_id = k3.id
where cs.id = :repo_id
""",
repo_id=int(repo_id))
if keys:
ssl_set = get_single_ssl_set(
keys, check_dates=self.check_ssl_dates)
if ssl_set:
plugin.set_ssl_options(ssl_set['ca_cert'],
ssl_set['client_cert'],
ssl_set['client_key'])
else:
raise ValueError(
"No valid SSL certificates were found for repository."
)
if not self.no_packages:
ret = self.import_packages(plugin, repo_id, url)
failed_packages += ret
self.import_groups(plugin, url)
if not self.no_errata:
self.import_updates(plugin, url)
# only for repos obtained from the DB
if self.sync_kickstart and repo_label:
try:
self.import_kickstart(plugin, repo_label)
except:
rhnSQL.rollback()
raise
except Exception:
e = sys.exc_info()[1]
log2(0, 0, "ERROR: %s" % e, stream=sys.stderr)
log2disk(0, "ERROR: %s" % e)
# pylint: disable=W0104
sync_error = -1
if plugin is not None:
plugin.clear_ssl_cache()
# Update cache with package checksums
rhnCache.set(checksum_cache_filename, self.checksum_cache)
if self.regen:
taskomatic.add_to_repodata_queue_for_channel_package_subscription(
[self.channel_label], [], "server.app.yumreposync")
taskomatic.add_to_erratacache_queue(self.channel_label)
self.update_date()
rhnSQL.commit()
# update permissions
fileutils.createPath(os.path.join(
CFG.MOUNT_POINT,
'rhn')) # if the directory exists update ownership only
for root, dirs, files in os.walk(os.path.join(CFG.MOUNT_POINT, 'rhn')):
for d in dirs:
fileutils.setPermsPath(os.path.join(root, d), group='apache')
for f in files:
fileutils.setPermsPath(os.path.join(root, f), group='apache')
elapsed_time = datetime.now() - start_time
log(
0, "Sync of channel completed in %s." %
str(elapsed_time).split('.')[0])
# if there is no global problems, but some packages weren't synced
if sync_error == 0 and failed_packages > 0:
sync_error = failed_packages
return elapsed_time, sync_error
raise
# If we got here, it means the directory exists
# file does not exist, attempt to create it
# we pass most of the exceptions through
try:
fd = os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0644)
except OSError, e:
# The file may be already there
if e.errno == EEXIST and os.access(fname, os.F_OK):
# Retry
continue
# Pass exception through
raise
# If we got here, the file is created, so break out of the loop
setPermsPath(fname, user, group, mode)
return fd
# Ran out of tries; something is fishy
# (if we manage to create or truncate the file, we've returned from the
# function already)
raise RuntimeError, "Attempt to create file %s failed" % fname
class LockedFile(object):
def __init__(self, name, modified = None, user='******', group='root', \
mode=0755):
if modified:
self.modified = timestamp(modified)
else:
def _safe_create(fname, user, group, mode):
""" This function returns a file descriptor for the open file fname
If the file is already there, it is truncated
otherwise, all the directories up to it are created and the file is created
as well.
"""
# There can be race conditions between the moment we check for the file
# existence and when we actually create it, so retry if something fails
tries = 5
while tries:
tries = tries - 1
# we're really picky about what can we do
if os.access(fname, os.F_OK): # file exists
if not os.access(fname, os.R_OK | os.W_OK):
raise UnreadableFileError()
fd = os.open(fname, os.O_WRONLY | os.O_TRUNC)
# We're done
return fd
# If directory does not exist, attempt to create it
dirname = os.path.dirname(fname)
if not os.path.isdir(dirname):
try:
#os.makedirs(dirname, 0755)
makedirs(dirname, mode, user, group)
except OSError:
e = sys.exc_info()[1]
# There is a window between the moment we check the disk and
# the one we try to create the directory
# We double-check the file existance here
if not (e.errno == EEXIST and os.path.isdir(dirname)):
# If the exception was thrown on a parent dir
# check the subdirectory to go through next loop.
if os.path.isdir(e.filename):
continue
# Pass exception through
raise
except:
# Pass exception through
raise
# If we got here, it means the directory exists
# file does not exist, attempt to create it
# we pass most of the exceptions through
try:
fd = os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL, int('0644', 8))
except OSError:
e = sys.exc_info()[1]
# The file may be already there
if e.errno == EEXIST and os.access(fname, os.F_OK):
# Retry
continue
# Pass exception through
raise
# If we got here, the file is created, so break out of the loop
setPermsPath(fname, user, group, mode)
return fd
# Ran out of tries; something is fishy
# (if we manage to create or truncate the file, we've returned from the
# function already)
raise RuntimeError("Attempt to create file %s failed" % fname)
raise
# If we got here, it means the directory exists
# file does not exist, attempt to create it
# we pass most of the exceptions through
try:
fd = os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0644)
except OSError, e:
# The file may be already there
if e.errno == EEXIST and os.access(fname, os.F_OK):
# Retry
continue
# Pass exception through
raise
# If we got here, the file is created, so break out of the loop
setPermsPath(fname, user, group, mode)
return fd
# Ran out of tries; something is fishy
# (if we manage to create or truncate the file, we've returned from the
# function already)
raise RuntimeError, "Attempt to create file %s failed" % fname
class LockedFile(object):
def __init__(self, name, modified = None, user='******', group='root', \
mode=0755):
if modified:
self.modified = timestamp(modified)
else:
