def checkout_project_pkgmeta(self, project, try_again=True):
""" Checks out the packages metadata of all packages in a project. """
project_dir = os.path.join(self.dest_dir, project)
util.safe_mkdir_p(project_dir)
filename = os.path.join(project_dir, "_pkgmeta")
tmpfilename = filename + ".new"
try:
url = osc_copy.makeurl(
self.conf.apiurl, ["search", "package"], ["match=%s" % urllib.quote("@project='%s'" % project)]
)
length = self._download_url_to_file(url, tmpfilename)
if length == 0:
# metadata files should never be empty
if try_again:
util.safe_unlink(tmpfilename)
return self.checkout_project_pkgmeta(project, False)
os.rename(tmpfilename, filename)
except (urllib2.HTTPError, urllib2.URLError, socket.error), e:
util.safe_unlink(tmpfilename)
if type(e) == urllib2.HTTPError and e.code == 404:
print >> sys.stderr, "Project %s doesn't exist." % (project,)
elif try_again:
self.checkout_project_pkgmeta(project, False)
else:
print >> sys.stderr, "Cannot get packages metadata of %s: %s" % (project, e)
return
def _get_packages_in_project(self, project, try_again=True):
project_dir = os.path.join(self.dest_dir, project)
util.safe_mkdir_p(project_dir)
filename = os.path.join(project_dir, "_pkglist")
try:
url = osc_copy.makeurl(self.conf.apiurl, ["public", "source", project])
length = self._download_url_to_file(url, filename)
if length == 0:
# metadata files should never be empty
if try_again:
util.safe_unlink(filename)
return self._get_packages_in_project(project, False)
except (urllib2.HTTPError, urllib2.URLError, socket.error), e:
util.safe_unlink(filename)
if type(e) == urllib2.HTTPError and e.code == 404:
return (None, "Project %s doesn't exist." % (project,))
elif try_again:
return self._get_packages_in_project(project, False)
else:
return (None, str(e))
def _get_packages_in_project(self, project, try_again=True):
project_dir = os.path.join(self.dest_dir, project)
util.safe_mkdir_p(project_dir)
filename = os.path.join(project_dir, '_pkglist')
try:
url = osc_copy.makeurl(self.conf.apiurl,
['public', 'source', project])
length = self._download_url_to_file(url, filename)
if length == 0:
# metadata files should never be empty
if try_again:
util.safe_unlink(filename)
return self._get_packages_in_project(project, False)
except (urllib2.HTTPError, urllib2.URLError, socket.error), e:
util.safe_unlink(filename)
if type(e) == urllib2.HTTPError and e.code == 404:
return (None, 'Project %s doesn\'t exist.' % (project, ))
elif try_again:
return self._get_packages_in_project(project, False)
else:
return (None, str(e))
def check_project(self, project, try_again=True):
""" Checks if the current checkout of a project is up-to-date, and queue task if necessary. """
project_dir = os.path.join(self.dest_dir, project)
util.safe_mkdir_p(project_dir)
filename = os.path.join(project_dir, "_status")
try:
url = osc_copy.makeurl(self.conf.apiurl, ["status", "project", project])
length = self._download_url_to_file(url, filename)
if length == 0:
# metadata files should never be empty
if try_again:
util.safe_unlink(filename)
return self.check_project(project, False)
except (urllib2.HTTPError, urllib2.URLError, socket.error), e:
util.safe_unlink(filename)
if type(e) == urllib2.HTTPError:
if e.code == 404:
print >> sys.stderr, "Project %s doesn't exist." % (project,)
elif e.code == 400:
# the status page doesn't always work :/
self.queue_checkout_project(project, primary=False, force_simple_checkout=True, no_config=True)
elif try_again:
self.check_project(project, False)
else:
print >> sys.stderr, "Cannot get status of %s: %s" % (project, e)
return
def checkout_project_pkgmeta(self, project, try_again=True):
""" Checks out the packages metadata of all packages in a project. """
project_dir = os.path.join(self.dest_dir, project)
util.safe_mkdir_p(project_dir)
filename = os.path.join(project_dir, '_pkgmeta')
tmpfilename = filename + '.new'
try:
url = osc_copy.makeurl(
self.conf.apiurl, ['search', 'package'],
['match=%s' % urllib.quote('@project=\'%s\'' % project)])
length = self._download_url_to_file(url, tmpfilename)
if length == 0:
# metadata files should never be empty
if try_again:
util.safe_unlink(tmpfilename)
return self.checkout_project_pkgmeta(project, False)
os.rename(tmpfilename, filename)
except (urllib2.HTTPError, urllib2.URLError, socket.error), e:
util.safe_unlink(tmpfilename)
if type(e) == urllib2.HTTPError and e.code == 404:
print >> sys.stderr, 'Project %s doesn\'t exist.' % (project, )
elif try_again:
self.checkout_project_pkgmeta(project, False)
else:
print >> sys.stderr, 'Cannot get packages metadata of %s: %s' % (
project, e)
return
def _get_file(self,
project,
package,
filename,
size,
revision=None,
try_again=True):
""" Download a file of a package. """
package_dir = os.path.join(self.dest_dir, project, package)
destfile = os.path.join(package_dir, filename)
tmpdestfile = destfile + '.new'
try:
query = None
if revision:
query = {'rev': revision}
url = osc_copy.makeurl(self.conf.apiurl, [
'public', 'source', project, package,
urllib.request.pathname2url(filename)
],
query=query)
length = self._download_url_to_file(url, tmpdestfile)
if length != size:
if try_again:
util.safe_unlink(tmpdestfile)
return self._get_file(project, package, filename, size,
revision, False)
os.rename(tmpdestfile, destfile)
except (urllib.error.HTTPError, urllib.error.URLError,
socket.error) as e:
util.safe_unlink(tmpdestfile)
if type(e) == urllib.error.HTTPError and e.code == 404:
print('File in package %s of project %s doesn\'t exist.' %
(filename, package, project),
file=sys.stderr)
elif try_again:
self._get_file(project, package, filename, size, revision,
False)
else:
print(
'Cannot get file %s for %s from %s: %s (queueing for next run)'
% (filename, package, project, e),
file=sys.stderr)
self.error_queue.put((project, package))
return
def _get_files_metadata(self,
project,
package,
save_basename,
revision=None,
try_again=True):
""" Download the file list of a package. """
package_dir = os.path.join(self.dest_dir, project, package)
filename = os.path.join(package_dir, save_basename)
tmpfilename = filename + '.new'
# download files metadata
try:
query = None
if revision:
query = {'rev': revision}
url = osc_copy.makeurl(self.conf.apiurl,
['public', 'source', project, package],
query=query)
length = self._download_url_to_file(url, tmpfilename)
if length == 0:
# metadata files should never be empty
if try_again:
util.safe_unlink(tmpfilename)
return self._get_files_metadata(project, package,
save_basename, revision,
False)
os.rename(tmpfilename, filename)
except (urllib2.HTTPError, urllib2.URLError, socket.error), e:
util.safe_unlink(tmpfilename)
if type(e) == urllib2.HTTPError and e.code == 404:
print >> sys.stderr, 'Package %s doesn\'t exist in %s.' % (
package, project)
elif try_again:
return self._get_files_metadata(project, package,
save_basename, revision, False)
elif revision:
print >> sys.stderr, 'Cannot download file list of %s from %s with specified revision: %s' % (
package, project, e)
else:
print >> sys.stderr, 'Cannot download file list of %s from %s: %s (queueing for next run)' % (
package, project, e)
self.error_queue.put((project, package))
return None
def _get_files_metadata(self, project, package, save_basename, revision=None, try_again=True):
""" Download the file list of a package. """
package_dir = os.path.join(self.dest_dir, project, package)
filename = os.path.join(package_dir, save_basename)
tmpfilename = filename + ".new"
# download files metadata
try:
query = None
if revision:
query = {"rev": revision}
url = osc_copy.makeurl(self.conf.apiurl, ["public", "source", project, package], query=query)
length = self._download_url_to_file(url, tmpfilename)
if length == 0:
# metadata files should never be empty
if try_again:
util.safe_unlink(tmpfilename)
return self._get_files_metadata(project, package, save_basename, revision, False)
os.rename(tmpfilename, filename)
except (urllib2.HTTPError, urllib2.URLError, socket.error), e:
util.safe_unlink(tmpfilename)
if type(e) == urllib2.HTTPError and e.code == 404:
print >> sys.stderr, "Package %s doesn't exist in %s." % (package, project)
elif try_again:
return self._get_files_metadata(project, package, save_basename, revision, False)
elif revision:
print >> sys.stderr, "Cannot download file list of %s from %s with specified revision: %s" % (
package,
project,
e,
)
else:
print >> sys.stderr, "Cannot download file list of %s from %s: %s (queueing for next run)" % (
package,
project,
e,
)
self.error_queue.put((project, package))
return None
def _get_packages_in_project(self, project, try_again=True):
project_dir = os.path.join(self.dest_dir, project)
util.safe_mkdir_p(project_dir)
filename = os.path.join(project_dir, '_pkglist')
try:
url = osc_copy.makeurl(self.conf.apiurl,
['public', 'source', project])
length = self._download_url_to_file(url, filename)
if length == 0:
# metadata files should never be empty
if try_again:
util.safe_unlink(filename)
return self._get_packages_in_project(project, False)
except (urllib.error.HTTPError, urllib.error.URLError,
socket.error) as e:
util.safe_unlink(filename)
if type(e) == urllib.error.HTTPError and e.code == 404:
return (None, 'Project %s doesn\'t exist.' % (project, ))
elif try_again:
return self._get_packages_in_project(project, False)
else:
return (None, str(e))
try:
root = ET.parse(filename).getroot()
except SyntaxError as e:
util.safe_unlink(filename)
if try_again:
return self._get_packages_in_project(project, False)
else:
return (None, 'Cannot parse list of packages in %s: %s' %
(project, e))
packages = [node.get('name') for node in root.findall('entry')]
util.safe_unlink(filename)
return (packages, None)
def checkout_package_meta(self, project, package, try_again=True):
""" Checks out the metadata of a package.
If we're interested in devel projects of this project, and the
devel package is not in a checked out devel project, then we queue
a checkout of this devel project.
"""
package_dir = os.path.join(self.dest_dir, project, package)
util.safe_mkdir_p(package_dir)
filename = os.path.join(package_dir, "_meta")
tmpfilename = filename + ".new"
try:
url = osc_copy.makeurl(self.conf.apiurl, ["public", "source", project, package, "_meta"])
length = self._download_url_to_file(url, tmpfilename)
if length == 0:
# metadata files should never be empty
if try_again:
util.safe_unlink(tmpfilename)
return self.checkout_package_meta(project, package, False)
os.rename(tmpfilename, filename)
except (urllib2.HTTPError, urllib2.URLError, socket.error), e:
util.safe_unlink(tmpfilename)
if type(e) == urllib2.HTTPError and e.code == 404:
print >> sys.stderr, "Package %s of project %s doesn't exist." % (package, project)
elif try_again:
self.checkout_package_meta(project, package, False)
else:
print >> sys.stderr, "Cannot get metadata of package %s in %s: %s (queueing for next run)" % (
package,
project,
e,
)
self.error_queue.put((project, package))
return
def checkout_package_meta(self, project, package, try_again=True):
""" Checks out the metadata of a package.
If we're interested in devel projects of this project, and the
devel package is not in a checked out devel project, then we queue
a checkout of this devel project.
"""
package_dir = os.path.join(self.dest_dir, project, package)
util.safe_mkdir_p(package_dir)
filename = os.path.join(package_dir, '_meta')
tmpfilename = filename + '.new'
try:
url = osc_copy.makeurl(
self.conf.apiurl,
['public', 'source', project, package, '_meta'])
length = self._download_url_to_file(url, tmpfilename)
if length == 0:
# metadata files should never be empty
if try_again:
util.safe_unlink(tmpfilename)
return self.checkout_package_meta(project, package, False)
os.rename(tmpfilename, filename)
except (urllib2.HTTPError, urllib2.URLError, socket.error), e:
util.safe_unlink(tmpfilename)
if type(e) == urllib2.HTTPError and e.code == 404:
print >> sys.stderr, 'Package %s of project %s doesn\'t exist.' % (
package, project)
elif try_again:
self.checkout_package_meta(project, package, False)
else:
print >> sys.stderr, 'Cannot get metadata of package %s in %s: %s (queueing for next run)' % (
package, project, e)
self.error_queue.put((project, package))
return
def _get_file(self, project, package, filename, size, revision=None, try_again=True):
""" Download a file of a package. """
package_dir = os.path.join(self.dest_dir, project, package)
destfile = os.path.join(package_dir, filename)
tmpdestfile = destfile + ".new"
try:
query = None
if revision:
query = {"rev": revision}
url = osc_copy.makeurl(
self.conf.apiurl, ["public", "source", project, package, urllib.pathname2url(filename)], query=query
)
length = self._download_url_to_file(url, tmpdestfile)
if length != size:
if try_again:
util.safe_unlink(tmpdestfile)
return self._get_file(project, package, filename, size, revision, False)
os.rename(tmpdestfile, destfile)
except (urllib2.HTTPError, urllib2.URLError, socket.error), e:
util.safe_unlink(tmpdestfile)
if type(e) == urllib2.HTTPError and e.code == 404:
print >> sys.stderr, "File in package %s of project %s doesn't exist." % (filename, package, project)
elif try_again:
self._get_file(project, package, filename, size, revision, False)
else:
print >> sys.stderr, "Cannot get file %s for %s from %s: %s (queueing for next run)" % (
filename,
package,
project,
e,
)
self.error_queue.put((project, package))
return
def check_project(self, project, try_again=True):
""" Checks if the current checkout of a project is up-to-date, and queue task if necessary. """
project_dir = os.path.join(self.dest_dir, project)
util.safe_mkdir_p(project_dir)
filename = os.path.join(project_dir, '_status')
try:
url = osc_copy.makeurl(self.conf.apiurl,
['status', 'project', project])
length = self._download_url_to_file(url, filename)
if length == 0:
# metadata files should never be empty
if try_again:
util.safe_unlink(filename)
return self.check_project(project, False)
except (urllib2.HTTPError, urllib2.URLError, socket.error), e:
util.safe_unlink(filename)
if type(e) == urllib2.HTTPError:
if e.code == 404:
print >> sys.stderr, 'Project %s doesn\'t exist.' % (
project, )
elif e.code == 400:
# the status page doesn't always work :/
self.queue_checkout_project(project,
primary=False,
force_simple_checkout=True,
no_config=True)
elif try_again:
self.check_project(project, False)
else:
print >> sys.stderr, 'Cannot get status of %s: %s' % (project,
e)
return
def check_project(self, project, try_again=True):
""" Checks if the current checkout of a project is up-to-date, and queue task if necessary. """
project_dir = os.path.join(self.dest_dir, project)
util.safe_mkdir_p(project_dir)
filename = os.path.join(project_dir, '_status')
try:
url = osc_copy.makeurl(self.conf.apiurl,
['status', 'project', project])
length = self._download_url_to_file(url, filename)
if length == 0:
# metadata files should never be empty
if try_again:
util.safe_unlink(filename)
return self.check_project(project, False)
except (urllib.error.HTTPError, urllib.error.URLError,
socket.error) as e:
util.safe_unlink(filename)
if type(e) == urllib.error.HTTPError:
if e.code == 404:
print('Project %s doesn\'t exist.' % (project, ),
file=sys.stderr)
elif e.code == 400:
# the status page doesn't always work :/
self.queue_checkout_project(project,
primary=False,
force_simple_checkout=True,
no_config=True)
elif try_again:
self.check_project(project, False)
else:
print('Cannot get status of %s: %s' % (project, e),
file=sys.stderr)
return
try:
packages_node = ET.parse(filename).getroot()
except SyntaxError as e:
util.safe_unlink(filename)
if try_again:
return self.check_project(project, False)
else:
print('Cannot parse status of %s: %s' % (project, e),
file=sys.stderr)
return
# We will have to remove all subdirectories that just don't belong to
# this project anymore.
subdirs_to_remove = [
file for file in os.listdir(project_dir)
if os.path.isdir(os.path.join(project_dir, file))
]
# Here's what we check to know if a package needs to be checked out again:
# - if there's no subdir
# - if it's a link:
# - check that the md5 from the status is the xsrcmd5 from the file
# list
# - check that we have _files-expanded and that all spec files are
# checked out
# - if it's not a link: check that the md5 from the status is the
# srcmd5 from the file list
for node in packages_node.findall('package'):
name = node.get('name')
srcmd5 = node.get('srcmd5')
is_link = len(node.findall('link')) > 0
try:
subdirs_to_remove.remove(name)
except ValueError:
pass
files = os.path.join(project_dir, name, '_files')
if not os.path.exists(files):
self.queue_checkout_package(project, name, primary=False)
continue
try:
files_root = ET.parse(files).getroot()
except SyntaxError:
self.queue_checkout_package(project, name, primary=False)
continue
if is_link:
previous_srcmd5 = files_root.get('xsrcmd5')
else:
previous_srcmd5 = files_root.get('srcmd5')
if srcmd5 != previous_srcmd5:
self.queue_checkout_package(project, name, primary=False)
# make sure we have all spec files
if is_link:
# for links, we open the list of files when expanded
files = os.path.join(project_dir, name, '_files-expanded')
if not os.path.exists(files):
self.queue_checkout_package(project, name, primary=False)
continue
try:
files_root = ET.parse(files).getroot()
except SyntaxError:
self.queue_checkout_package(project, name, primary=False)
continue
cont = False
for entry in files_root.findall('entry'):
filename = entry.get('name')
if filename.endswith('.spec'):
specfile = os.path.join(project_dir, name, filename)
if not os.path.exists(specfile):
self.queue_checkout_package(project,
name,
primary=False)
cont = True
break
if cont:
continue
# Remove useless subdirectories
for subdir in subdirs_to_remove:
shutil.rmtree(os.path.join(project_dir, subdir))
util.safe_unlink(filename)
def checkout_package_meta(self, project, package, try_again=True):
""" Checks out the metadata of a package.
If we're interested in devel projects of this project, and the
devel package is not in a checked out devel project, then we queue
a checkout of this devel project.
"""
package_dir = os.path.join(self.dest_dir, project, package)
util.safe_mkdir_p(package_dir)
filename = os.path.join(package_dir, '_meta')
tmpfilename = filename + '.new'
try:
url = osc_copy.makeurl(
self.conf.apiurl,
['public', 'source', project, package, '_meta'])
length = self._download_url_to_file(url, tmpfilename)
if length == 0:
# metadata files should never be empty
if try_again:
util.safe_unlink(tmpfilename)
return self.checkout_package_meta(project, package, False)
os.rename(tmpfilename, filename)
except (urllib.error.HTTPError, urllib.error.URLError,
socket.error) as e:
util.safe_unlink(tmpfilename)
if type(e) == urllib.error.HTTPError and e.code == 404:
print('Package %s of project %s doesn\'t exist.' %
(package, project),
file=sys.stderr)
elif try_again:
self.checkout_package_meta(project, package, False)
else:
print(
'Cannot get metadata of package %s in %s: %s (queueing for next run)'
% (package, project, e),
file=sys.stderr)
self.error_queue.put((project, package))
return
# Are we interested in devel projects of this project, and if yes,
# should we check out the devel project if needed?
if project not in self.conf.projects:
return
if not self.conf.projects[project].checkout_devel_projects:
return
try:
package_node = ET.parse(filename).getroot()
except SyntaxError:
return
devel_node = package_node.find('devel')
if devel_node is None:
return
devel_project = devel_node.get('project')
project_dir = os.path.join(self.dest_dir, devel_project)
if not os.path.exists(project_dir):
self.queue_checkout_project(devel_project,
parent=project,
primary=False)