def create_adi_project(self, name):
"""Create an ADI project."""
if not name:
name = self._candidate_adi_project()
else:
name = self.adi_prj_from_number(name)
adi_projects = self.get_adi_projects()
if name in adi_projects:
raise Exception('Project {} already exist'.format(name))
meta = """
<project name="{}">
<title></title>
<description></description>
<publish>
<disable/>
</publish>
<debuginfo>
<enable/>
</debuginfo>
<repository name="standard">
<path project="{}" repository="standard"/>
<arch>x86_64</arch>
</repository>
</project>""".format(name, self.project)
url = make_meta_url('prj', name, self.apiurl)
http_PUT(url, data=meta)
# put twice because on first put, the API adds useless maintainer
http_PUT(url, data=meta)
return name
def set_prj_pseudometa(self, project, meta):
"""
Sets project description to the YAML of the provided object
:param project: project to save into
:param meta: data to save
"""
# Get current metadata
url = make_meta_url('prj', project, self.apiurl)
root = ET.parse(http_GET(url)).getroot()
# Find description
description = root.find('description')
# Order the requests and replace it with yaml
meta['requests'] = sorted(meta['requests'], key=lambda x: x['id'])
description.text = yaml.dump(meta)
# Find title
title = root.find('title')
# Put something nice into title as well
new_title = []
for request in meta['requests']:
new_title.append(request['package'])
nt = ', '.join(sorted(new_title))
title.text = nt[:240]
# Write XML back
url = make_meta_url('prj', project, self.apiurl, force=True)
http_PUT(url, data=ET.tostring(root))
# Invalidate here the cache for this stating project
self._invalidate_get_prj_pseudometa(project)
def switch_flag_in_prj(self,
project,
flag='build',
state='disable',
repository=None,
arch=None):
url = self.makeurl(['source', project, '_meta'])
prjmeta = ET.parse(http_GET(url)).getroot()
flagxml = prjmeta.find(flag)
if not flagxml: # appending is fine
flagxml = ET.SubElement(prjmeta, flag)
foundone = False
for build in flagxml:
if build.get('repository', None) == repository and build.get(
'arch', None) == arch:
build.tag = state
foundone = True
# need to add a global one
if not foundone:
query = {}
if arch:
query['arch'] = arch
if repository:
query['repository'] = repository
ET.SubElement(flagxml, state, query)
http_PUT(url, data=ET.tostring(prjmeta))
def update_factory_version(self):
"""Update project (Factory, 13.2, ...) version if is necessary."""
# XXX TODO - This method have `factory` in the name. Can be
# missleading.
project = self.api.project
curr_version = date.today().strftime('%Y%m%d')
url = self.api.makeurl(['source', project], {'view': 'productlist'})
products = ET.parse(http_GET(url)).getroot()
for product in products.findall('product'):
product_name = product.get('name') + '.product'
product_pkg = product.get('originpackage')
url = self.api.makeurl(
['source', project, product_pkg, product_name])
product_spec = http_GET(url).read()
new_product = re.sub(r'<version>\d{8}</version>',
'<version>%s</version>' % curr_version,
product_spec)
if product_spec != new_product:
http_PUT(url + '?comment=Update+version', data=new_product)
service = {'cmd': 'runservice'}
ports_prjs = ['PowerPC', 'ARM', 'zSystems']
for ports in ports_prjs:
project = self.api.project + ':' + ports
if self.api.item_exists(project):
baseurl = ['source', project, '000product']
url = self.api.makeurl(baseurl, query=service)
self.api.retried_POST(url)
def create_adi_project(self, name):
"""Create an ADI project."""
if not name:
name = self._candidate_adi_project()
else:
name = self.adi_prj_from_number(name)
adi_projects = self.get_adi_projects()
if name in adi_projects:
raise Exception('Project {} already exist'.format(name))
meta = """
<project name="{}">
<title></title>
<description></description>
<publish>
<disable/>
</publish>
<debuginfo>
<enable/>
</debuginfo>
<repository name="standard">
<path project="{}" repository="standard"/>
<arch>x86_64</arch>
</repository>
</project>""".format(name, self.project)
url = make_meta_url('prj', name, self.apiurl)
http_PUT(url, data=meta)
# put twice because on first put, the API adds useless maintainer
http_PUT(url, data=meta)
return name
def project_clone(apiurl_source, apiurl_target, project):
users_clone(apiurl_source, apiurl_target, project)
project_workaround(project)
# Write stripped version that does not include repos with path references.
url = makeurl(apiurl_target, ['source', project.get('name'), '_meta'])
stripped = deepcopy(project)
project_references_remove(stripped)
http_PUT(url, data=ET.tostring(stripped))
for link in project.xpath('link[@project]'):
if not project_fence(link.get('project')):
project.remove(link)
break
# Valid reference to project and thus should be cloned.
path = ['source', link.get('project'), '_meta']
entity_clone(apiurl_source, apiurl_target, path, clone=project_clone)
# Clone projects referenced in repository paths.
for repository in project.findall('repository'):
for target in repository.xpath('./path') + repository.xpath('./releasetarget'):
if not project_fence(target.get('project')):
project.remove(repository)
break
# Valid reference to project and thus should be cloned.
path = ['source', target.get('project'), '_meta']
entity_clone(apiurl_source, apiurl_target, path, clone=project_clone)
def source_file_save(apiurl, project, package, filename, content, comment=None):
if not comment:
comment = 'update by OSRT tools'
comment += ' (host {})'.format(socket.gethostname())
url = makeurl(apiurl, ['source', project, package, filename], {'comment': comment})
http_PUT(url, data=content)
def entity_clone(apiurl_source, apiurl_target, path, sanitize=None, clone=None, after=None):
if not hasattr(entity_clone, 'cloned'):
entity_clone.cloned = []
if path[0] == 'source' and not project_fence(path[1]):
# Skip projects outside of fence by marking as cloned.
if path not in entity_clone.cloned:
entity_clone.cloned.append(path)
if path in entity_clone.cloned:
print('skip {}'.format('/'.join(path)))
return
print('clone {}'.format('/'.join(path)))
entity_clone.cloned.append(path)
url = makeurl(apiurl_source, path)
entity = ET.parse(http_GET(url)).getroot()
if sanitize:
sanitize(entity)
if clone:
clone(apiurl_source, apiurl_target, entity)
url = makeurl(apiurl_target, path)
http_PUT(url, data=ET.tostring(entity))
if after:
after(apiurl_source, apiurl_target, entity)
def update_factory_version(self):
"""Update project (Factory, 13.2, ...) version if is necessary."""
# XXX TODO - This method have `factory` in the name. Can be
# missleading.
# If thereis not product defined for this project, show the
# warning and return.
if not self.api.cproduct:
warnings.warn('There is not product defined in the configuration file.')
return
project = self.api.project
url = self.api.makeurl(['source', project, '_product', self.api.cproduct])
product = http_GET(url).read()
curr_version = date.today().strftime('%Y%m%d')
new_product = re.sub(r'<version>\d{8}</version>', '<version>%s</version>' % curr_version, product)
if product != new_product:
http_PUT(url + '?comment=Update+version', data=new_product)
service = {'cmd': 'runservice'}
ports_prjs = ['PowerPC', 'ARM' ]
for ports in ports_prjs:
project = self.api.project + ':' + ports
if self.api.item_exists(project):
baseurl = ['source', project, '_product']
url = self.api.makeurl(baseurl, query=service)
self.api.retried_POST(url)
def entity_clone(apiurl_source, apiurl_target, path, sanitize=None, clone=None, after=None):
if not hasattr(entity_clone, 'cloned'):
entity_clone.cloned = []
if path[0] == 'source' and not project_fence(path[1]):
# Skip projects outside of fence by marking as cloned.
if path not in entity_clone.cloned:
entity_clone.cloned.append(path)
if path in entity_clone.cloned:
print('skip {}'.format('/'.join(path)))
return
print('clone {}'.format('/'.join(path)))
entity_clone.cloned.append(path)
url = makeurl(apiurl_source, path)
entity = ET.parse(http_GET(url)).getroot()
if sanitize:
sanitize(entity)
if clone:
clone(apiurl_source, apiurl_target, entity)
url = makeurl(apiurl_target, path)
http_PUT(url, data=ET.tostring(entity))
if after:
after(apiurl_source, apiurl_target, entity)
def set_prj_pseudometa(self, project, meta):
"""
Sets project description to the YAML of the provided object
:param project: project to save into
:param meta: data to save
"""
# Get current metadata
url = make_meta_url('prj', project, self.apiurl)
root = ET.parse(http_GET(url)).getroot()
# Find description
description = root.find('description')
# Order the requests and replace it with yaml
meta['requests'] = sorted(meta['requests'], key=lambda x: x['id'])
description.text = yaml.dump(meta)
# Find title
title = root.find('title')
# Put something nice into title as well
new_title = []
for request in meta['requests']:
new_title.append(request['package'])
nt = ', '.join(sorted(new_title))
title.text = nt[:240]
# Write XML back
url = make_meta_url('prj', project, self.apiurl, force=True)
http_PUT(url, data=ET.tostring(root))
def update_factory_version(self):
"""Update project (Factory, 13.2, ...) version if is necessary."""
# XXX TODO - This method have `factory` in the name. Can be
# missleading.
project = self.api.project
curr_version = date.today().strftime('%Y%m%d')
update_version_attr = False
url = self.api.makeurl(['source', project], {'view': 'productlist'})
products = ET.parse(http_GET(url)).getroot()
for product in products.findall('product'):
product_name = product.get('name') + '.product'
product_pkg = product.get('originpackage')
url = self.api.makeurl(['source', project, product_pkg, product_name])
product_spec = http_GET(url).read()
new_product = re.sub(r'<version>\d{8}</version>', '<version>%s</version>' % curr_version, product_spec)
if product_spec != new_product:
update_version_attr = True
http_PUT(url + '?comment=Update+version', data=new_product)
if update_version_attr:
self.update_version_attribute(project, curr_version)
ports_prjs = ['PowerPC', 'ARM', 'zSystems' ]
for ports in ports_prjs:
project = self.api.project + ':' + ports
if self.api.item_exists(project) and update_version_attr:
self.update_version_attribute(project, curr_version)
def project_clone(apiurl_source, apiurl_target, project):
users_clone(apiurl_source, apiurl_target, project)
project_workaround(project)
# Write stripped version that does not include repos with path references.
url = makeurl(apiurl_target, ['source', project.get('name'), '_meta'])
stripped = deepcopy(project)
project_references_remove(stripped)
http_PUT(url, data=ET.tostring(stripped))
for link in project.xpath('link[@project]'):
if not project_fence(link.get('project')):
project.remove(link)
break
# Valid reference to project and thus should be cloned.
path = ['source', link.get('project'), '_meta']
entity_clone(apiurl_source, apiurl_target, path, clone=project_clone)
# Clone projects referenced in repository paths.
for repository in project.findall('repository'):
for target in repository.xpath('./path') + repository.xpath('./releasetarget'):
if not project_fence(target.get('project')):
project.remove(repository)
break
# Valid reference to project and thus should be cloned.
path = ['source', target.get('project'), '_meta']
entity_clone(apiurl_source, apiurl_target, path, clone=project_clone)
def update_factory_version(self):
"""Update project (Factory, 13.2, ...) version if is necessary."""
# XXX TODO - This method have `factory` in the name. Can be
# missleading.
# If thereis not product defined for this project, show the
# warning and return.
if not self.api.cproduct:
warnings.warn('There is not product defined in the configuration file.')
return
project = self.api.project
url = self.api.makeurl(['source', project, '_product', self.api.cproduct])
product = http_GET(url).read()
curr_version = date.today().strftime('%Y%m%d')
new_product = re.sub(r'<version>\d{8}</version>', '<version>%s</version>' % curr_version, product)
if product != new_product:
http_PUT(url + '?comment=Update+version', data=new_product)
service = {'cmd': 'runservice'}
ports_prjs = ['PowerPC', 'ARM' ]
for ports in ports_prjs:
project = self.api.project + ':' + ports
if self.api.item_exists(project):
baseurl = ['source', project, '_product']
url = self.api.makeurl(baseurl, query=service)
self.api.retried_POST(url)
def submit_to_prj(self, act, project, force_enable_build=False):
"""
Links sources from request to project
:param act: action for submit request
:param project: project to link into
:param force_enable_build: overwrite the ring criteria to enable
or disable the build
"""
src_prj = act.src_project
src_rev = act.src_rev
src_pkg = act.src_package
tar_pkg = act.tgt_package
disable_build = False
# The force_enable_build will avoid the
# map_ring_package_to_subproject
if not force_enable_build:
if self.crings and not self.ring_packages.get(tar_pkg) and not self.is_adi_project(project):
disable_build = True
logging.warning("{}/{} not in ring, build disabled".format(project, tar_pkg))
else:
project = self.map_ring_package_to_subject(project, tar_pkg)
self.create_package_container(project, tar_pkg,
disable_build=disable_build)
# expand the revision to a md5
url = self.makeurl(['source', src_prj, src_pkg],
{'rev': src_rev, 'expand': 1})
f = http_GET(url)
root = ET.parse(f).getroot()
src_rev = root.attrib['srcmd5']
src_vrev = root.attrib.get('vrev')
# link stuff - not using linkpac because linkpac copies meta
# from source
root = ET.Element('link', package=src_pkg, project=src_prj,
rev=src_rev)
if src_vrev:
root.attrib['vrev'] = src_vrev
url = self.makeurl(['source', project, tar_pkg, '_link'])
http_PUT(url, data=ET.tostring(root))
for sub_prj, sub_pkg in self.get_sub_packages(tar_pkg):
sub_prj = self.map_ring_package_to_subject(project, sub_pkg)
# print project, tar_pkg, sub_pkg, sub_prj
if sub_prj == project: # skip inner-project links
continue
self.create_package_container(sub_prj, sub_pkg)
root = ET.Element('link', package=tar_pkg, project=project)
url = self.makeurl(['source', sub_prj, sub_pkg, '_link'])
http_PUT(url, data=ET.tostring(root))
return tar_pkg
def save_file_content(self, project, package, filename, content):
"""
Save content to a project/package/file
:param project: The project containing the package
:param package: the package to update
:param filename: the filename to save the data to
:param content: the content to write to the file
"""
url = self.makeurl(['source', project, package, filename])
http_PUT(url + '?comment=scripted+update', data=content)
def save_file_content(self, project, package, filename, content):
"""
Save content to a project/package/file
:param project: The project containing the package
:param package: the package to update
:param filename: the filename to save the data to
:param content: the content to write to the file
"""
url = self.makeurl(['source', project, package, filename])
http_PUT(url + '?comment=scripted+update', data=content)
def submit_to_prj(self, act, project, force_enable_build=False):
"""
Links sources from request to project
:param act: action for submit request
:param project: project to link into
:param force_enable_build: overwrite the ring criteria to enable
or disable the build
"""
src_prj = act.src_project
src_rev = act.src_rev
src_pkg = act.src_package
tar_pkg = act.tgt_package
disable_build = False
# The force_enable_build will avoid the
# map_ring_package_to_subproject
if not force_enable_build:
if self.crings and not self.ring_packages.get(tar_pkg) and not self.is_adi_project(project):
disable_build = True
else:
project = self.map_ring_package_to_subject(project, tar_pkg)
self.create_package_container(project, tar_pkg,
disable_build=disable_build)
# expand the revision to a md5
url = self.makeurl(['source', src_prj, src_pkg],
{'rev': src_rev, 'expand': 1})
f = http_GET(url)
root = ET.parse(f).getroot()
src_rev = root.attrib['srcmd5']
src_vrev = root.attrib.get('vrev')
# link stuff - not using linkpac because linkpac copies meta
# from source
root = ET.Element('link', package=src_pkg, project=src_prj,
rev=src_rev)
if src_vrev:
root.attrib['vrev'] = src_vrev
url = self.makeurl(['source', project, tar_pkg, '_link'])
http_PUT(url, data=ET.tostring(root))
for sub_prj, sub_pkg in self.get_sub_packages(tar_pkg):
sub_prj = self.map_ring_package_to_subject(project, sub_pkg)
# print project, tar_pkg, sub_pkg, sub_prj
if sub_prj == project: # skip inner-project links
continue
self.create_package_container(sub_prj, sub_pkg)
root = ET.Element('link', package=tar_pkg, project=project)
url = self.makeurl(['source', sub_prj, sub_pkg, '_link'])
http_PUT(url, data=ET.tostring(root))
return tar_pkg
def sync(self):
hash = core.dgst(self.filename)
if self.change_is_required == True and hash == self.hash_orig:
os.unlink(self.filename)
return True
# don't do any exception handling... it's up to the caller what to do in case
# of an exception
core.http_PUT(self.url, file=self.filename)
os.unlink(self.filename)
return True
def sync(self):
hash = core.dgst(self.filename)
if self.change_is_required == True and hash == self.hash_orig:
os.unlink(self.filename)
return True
# don't do any exception handling... it's up to the caller what to do in case
# of an exception
core.http_PUT(self.url, file=self.filename)
os.unlink(self.filename)
return True
def update_factory_version(self):
"""Update openSUSE (Factory, 13.2, ...) version if is necessary."""
project = 'openSUSE:{}'.format(self.api.opensuse)
url = self.api.makeurl(['source', project, '_product', 'openSUSE.product'])
product = http_GET(url).read()
curr_version = date.today().strftime('%Y%m%d')
new_product = re.sub(r'<version>\d{8}</version>', '<version>%s</version>' % curr_version, product)
if product != new_product:
http_PUT(url + '?comment=Update+version', data=new_product)
def source_file_save(apiurl,
project,
package,
filename,
content,
comment=None):
if not comment:
comment = 'update by OSRT tools'
comment += ' (host {})'.format(socket.gethostname())
url = makeurl(apiurl, ['source', project, package, filename],
{'comment': comment})
http_PUT(url, data=content)
def __update_meta(self, project, providers, metatype):
"""Extracts a meta xml from rpm and uploads them to project.
:returns: uploaded pattern names and error messages
:rtype: tuple(list, list)
"""
uploaded = []
errors = []
for package, targets in providers.items():
for target, binaries in targets.items():
for binary in binaries:
with Lab(prefix=metatype) as lab:
# Download the rpm
try:
self.obs.getBinary(project, target, package,
binary, lab.real_path(binary))
except HTTPError as exc:
errors.append("Failed to download %s: HTTP %s %s" %
(binary, exc.code, exc.filename))
except Exception as exc:
errors.append("Failed to download %s: %s" % (binary, exc))
if errors:
return uploaded, errors
# Extract pattern (xml) files from the rpm
for xml in extract_rpm(lab.real_path(binary), lab.path,
["*.xml"]):
meta = os.path.basename(xml)
submetatype = os.path.basename(os.path.dirname(xml))
print(meta, metatype, submetatype)
try:
with open(lab.real_path(xml), 'r') as fd:
metadata = [line.replace("@PROJECT@", project) for line in fd.readlines()]
# Update meta
if submetatype == "aggregates":
pkgname = os.path.splitext(meta)[0]
core.edit_meta(metatype='pkg', path_args=(project, pkgname), template_args=({'name': pkgname, 'user': '******'}), apiurl=self.obs.apiurl)
u = core.makeurl(self.obs.apiurl, ['source', project, pkgname, '_aggregate'])
print u
print metadata
core.http_PUT(u, data="\n".join(metadata))
else:
core.edit_meta(metatype, project, data=metadata)
uploaded.append(metatype + '/' + meta)
except HTTPError as exc:
errors.append("Failed to upload %s:\nHTTP %s %s\n%s" %
(meta, exc.code, exc.filename,
exc.fp.read()))
except Exception as exc:
errors.append("Failed to upload %s: %s" %
(meta, exc))
return uploaded, errors
def retried_PUT(self, url, data):
try:
return http_PUT(url, data=data)
except urllib2.HTTPError, e:
if e.code / 100 == 5:
print 'Retrying {}'.format(url)
return self.retried_PUT(url, data)
raise e
def retried_PUT(self, url, data):
try:
return http_PUT(url, data=data)
except urllib2.HTTPError, e:
if e.code / 100 == 5:
print 'Retrying {}'.format(url)
return self.retried_PUT(url, data)
raise e
def create_package_container(self, project, package, disable_build=False):
"""
Creates a package container without any fields in project/package
:param project: project to create it
:param package: package name
:param disable_build: should the package be created with build
flag disabled
"""
dst_meta = '<package name="{}"><title/><description/></package>'
dst_meta = dst_meta.format(package)
if disable_build:
root = ET.fromstring(dst_meta)
elm = ET.SubElement(root, 'build')
ET.SubElement(elm, 'disable')
dst_meta = ET.tostring(root)
url = self.makeurl(['source', project, package, '_meta'])
http_PUT(url, data=dst_meta)
def reset_rebuild_data(self, project):
url = self.api.makeurl(['source', self.api.cstaging, 'dashboard', 'support_pkg_rebuild?expand=1'])
try:
data = http_GET(url)
except urllib2.HTTPError:
return
tree = ET.parse(data)
root = tree.getroot()
for stg in root.findall('staging'):
if stg.get('name') == project:
stg.find('rebuild').text = 'unknown'
stg.find('supportpkg').text = ''
# reset accpted staging project rebuild state to unknown and clean up
# supportpkg list
url = self.api.makeurl(['source', self.api.cstaging, 'dashboard', 'support_pkg_rebuild'])
content = ET.tostring(root)
http_PUT(url + '?comment=accept+command+update', data=content)
def create_package_container(self, project, package, disable_build=False):
"""
Creates a package container without any fields in project/package
:param project: project to create it
:param package: package name
:param disable_build: should the package be created with build
flag disabled
"""
dst_meta = '<package name="{}"><title/><description/></package>'
dst_meta = dst_meta.format(package)
if disable_build:
root = ET.fromstring(dst_meta)
elm = ET.SubElement(root, 'build')
ET.SubElement(elm, 'disable')
dst_meta = ET.tostring(root)
url = self.makeurl(['source', project, package, '_meta'])
http_PUT(url, data=dst_meta)
def delete_to_prj(self, act, project):
"""
Hides Package in project
:param act: action for delete request
:param project: project to hide in
"""
tar_pkg = act.tgt_package
project = self.map_ring_package_to_subject(project, tar_pkg)
self.create_and_wipe_package(project, tar_pkg)
for sub_prj, sub_pkg in self.get_sub_packages(tar_pkg):
sub_prj = self.map_ring_package_to_subject(project, sub_pkg)
self.create_and_wipe_package(sub_prj, sub_pkg)
# create a link so unselect can find it
root = ET.Element('link', package=tar_pkg, project=project)
url = self.makeurl(['source', sub_prj, sub_pkg, '_link'])
http_PUT(url, data=ET.tostring(root))
return tar_pkg
def delete_to_prj(self, act, project):
"""
Hides Package in project
:param act: action for delete request
:param project: project to hide in
"""
tar_pkg = act.tgt_package
project = self.map_ring_package_to_subject(project, tar_pkg)
self.create_and_wipe_package(project, tar_pkg)
for sub_prj, sub_pkg in self.get_sub_packages(tar_pkg):
sub_prj = self.map_ring_package_to_subject(project, sub_pkg)
self.create_and_wipe_package(sub_prj, sub_pkg)
# create a link so unselect can find it
root = ET.Element('link', package=tar_pkg, project=project)
url = self.makeurl(['source', sub_prj, sub_pkg, '_link'])
http_PUT(url, data=ET.tostring(root))
return tar_pkg
def update_factory_version(self):
"""Update project (Factory, 13.2, ...) version if is necessary."""
# XXX TODO - This method have `factory` in the name. Can be
# missleading.
project = self.api.project
curr_version = date.today().strftime('%Y%m%d')
update_version_attr = False
url = self.api.makeurl(['source', project], {'view': 'productlist'})
products = ET.parse(http_GET(url)).getroot()
for product in products.findall('product'):
product_name = product.get('name') + '.product'
product_pkg = product.get('originpackage')
product_spec = source_file_load(self.api.apiurl, project,
product_pkg, product_name)
new_product = re.sub(r'<version>\d{8}</version>',
'<version>%s</version>' % curr_version,
product_spec)
if product_spec != new_product:
update_version_attr = True
url = self.api.makeurl(
['source', project, product_pkg, product_name])
http_PUT(url + '?comment=Update+version', data=new_product)
if update_version_attr:
self.update_version_attribute(project, curr_version)
ports_prjs = ['PowerPC', 'ARM', 'zSystems']
for ports in ports_prjs:
project = self.api.project + ':' + ports
if self.api.item_exists(project) and update_version_attr:
self.update_version_attribute(project, curr_version)
def switch_flag_in_prj(self, project, flag='build', state='disable', repository=None, arch=None):
url = self.makeurl(['source', project, '_meta'])
prjmeta = ET.parse(http_GET(url)).getroot()
flagxml = prjmeta.find(flag)
if not flagxml: # appending is fine
flagxml = ET.SubElement(prjmeta, flag)
foundone = False
for build in flagxml:
if build.get('repository', None) == repository and build.get('arch', None) == arch:
build.tag = state
foundone = True
# need to add a global one
if not foundone:
query = {}
if arch:
query['arch'] = arch
if repository:
query['repository'] = repository
ET.SubElement(flagxml, state, query)
http_PUT(url, data=ET.tostring(prjmeta))
def handle_wi(self, wid):
""" Workitem handling function """
wid.result = False
f = wid.fields
p = wid.params
project = None
package = None
if f.project and f.package:
project = f.project
package = f.package
if p.project and p.package:
project = p.project
package = p.package
if not project or not package:
raise RuntimeError("Missing mandatory field or parameter: package, project")
if not f.repourl and not p.repourl:
raise RuntimeError("Missing mandatory field or parameter: repourl")
params = {}
if f.repourl:
params["url"] = f.repourl
if p.repourl:
params["url"] = p.repourl
params["service"], params["repo"] = find_service_repo(params["url"])
if f.branch:
params["branch"] = f.branch
if p.branch:
params["branch"] = p.branch
params["revision"] = ""
if f.revision:
params["revision"] = f.revision
if p.revision:
params["revision"] = p.revision
params["token"] = ""
params["debian"] = ""
params["dumb"] = ""
if f.token:
params["token"] = f.token
if p.token:
params["token"] = p.token
if p.debian:
params["debian"] = p.debian
if f.debian:
params["debian"] = f.debian
if p.dumb:
params["dumb"] = p.dumb
if f.dumb:
params["dumb"] = f.dumb
if "branch" in params and params["branch"].startswith("pkg-"):
if not "service" in params or not "repo" in params:
raise RuntimeError("Service/Repo not found in repourl %s " % p.repourl)
service = git_pkg_service
else:
service = tar_git_service
# the simple approach doesn't work with project links
#if self.obs.isNewPackage(project, package):
#self.obs.getCreatePackage(str(project), str(package))
#else:
try:
core.show_files_meta(self.obs.apiurl, str(project), str(package), expand=False, meta=True)
except Exception, exc:
data = core.metatypes['pkg']['template']
data = StringIO(data % { "name" : str(package), "user" : self.obs.getUserName() }).readlines()
u = core.makeurl(self.obs.apiurl, ['source', str(project), str(package), "_meta"])
x = core.http_PUT(u, data="".join(data))
def __update_meta(self, project, providers, metatype):
"""Extracts a meta xml from rpm and uploads them to project.
:returns: uploaded pattern names and error messages
:rtype: tuple(list, list)
"""
uploaded = []
errors = []
for package, targets in providers.items():
for target, binaries in targets.items():
for binary in binaries:
with Lab(prefix=metatype) as lab:
# Download the rpm
try:
self.obs.getBinary(project, target, package,
binary, lab.real_path(binary))
except HTTPError as exc:
errors.append("Failed to download %s: HTTP %s %s" %
(binary, exc.code, exc.filename))
except Exception as exc:
errors.append("Failed to download %s: %s" %
(binary, exc))
if errors:
return uploaded, errors
# Extract pattern (xml) files from the rpm
for xml in extract_rpm(lab.real_path(binary), lab.path,
["*.xml"]):
meta = os.path.basename(xml)
submetatype = os.path.basename(
os.path.dirname(xml))
print(meta, metatype, submetatype)
try:
with open(lab.real_path(xml), 'r') as fd:
metadata = [
line.replace("@PROJECT@", project)
for line in fd.readlines()
]
# Update meta
if submetatype == "aggregates":
pkgname = os.path.splitext(meta)[0]
core.edit_meta(metatype='pkg',
path_args=(project,
pkgname),
template_args=({
'name':
pkgname,
'user':
'******'
}),
apiurl=self.obs.apiurl)
u = core.makeurl(self.obs.apiurl, [
'source', project, pkgname,
'_aggregate'
])
print u
print metadata
core.http_PUT(u, data="\n".join(metadata))
else:
core.edit_meta(metatype,
project,
data=metadata)
uploaded.append(metatype + '/' + meta)
except HTTPError as exc:
errors.append(
"Failed to upload %s:\nHTTP %s %s\n%s" %
(meta, exc.code, exc.filename,
exc.fp.read()))
except Exception as exc:
errors.append("Failed to upload %s: %s" %
(meta, exc))
return uploaded, errors
def handle_wi(self, wid):
""" Workitem handling function """
wid.result = False
f = wid.fields
p = wid.params
project = None
package = None
if f.project and f.package:
project = f.project
package = f.package
if p.project and p.package:
project = p.project
package = p.package
if not project or not package:
raise RuntimeError(
"Missing mandatory field or parameter: package, project")
if not f.repourl and not p.repourl:
raise RuntimeError("Missing mandatory field or parameter: repourl")
params = {}
if f.repourl:
params["url"] = f.repourl
if p.repourl:
params["url"] = p.repourl
params["service"], params["repo"] = find_service_repo(params["url"])
if f.branch:
params["branch"] = f.branch
if p.branch:
params["branch"] = p.branch
params["revision"] = ""
if f.revision:
params["revision"] = f.revision
if p.revision:
params["revision"] = p.revision
params["token"] = ""
params["debian"] = ""
params["dumb"] = ""
if f.token:
params["token"] = f.token
if p.token:
params["token"] = p.token
if p.debian:
params["debian"] = p.debian
if f.debian:
params["debian"] = f.debian
if p.dumb:
params["dumb"] = p.dumb
if f.dumb:
params["dumb"] = f.dumb
if "branch" in params and params["branch"].startswith("pkg-"):
if not "service" in params or not "repo" in params:
raise RuntimeError(
"Service/Repo not found in repourl %s " % p.repourl)
service = git_pkg_service
else:
service = tar_git_service
# the simple approach doesn't work with project links
# if self.obs.isNewPackage(project, package):
# self.obs.getCreatePackage(str(project), str(package))
# else:
try:
pkginfo = core.show_files_meta(
self.obs.apiurl, str(project), str(package), expand=False, meta=True)
if "<entry" not in pkginfo:
# This is a link and it needs branching from the linked project
# so grab the meta and extract the project from the link
print "Found %s as a link in %s" % (package, project)
x = etree.fromstring(
"".join(core.show_project_meta(self.obs.apiurl, project)))
l = x.find('link')
if l is None:
raise Exception(
"Expected a <link> in project %s." % project)
print "Got a link %s" % l
linked_project = l.get('project')
print "Branching %s to overwrite _service" % package
core.branch_pkg(self.obs.apiurl, linked_project,
str(package), target_project=str(project))
except Exception, exc:
print "Doing a metatype pkg add because I caught %s" % exc
print "Creating package %s in project %s" % (package, project)
data = core.metatypes['pkg']['template']
data = StringIO(
data % {"name": str(package), "user": self.obs.getUserName()}).readlines()
u = core.makeurl(
self.obs.apiurl, ['source', str(project), str(package), "_meta"])
x = core.http_PUT(u, data="".join(data))
print "HTTP PUT result of pkg add : %s" % x
def handle_wi(self, wid):
""" Workitem handling function """
wid.result = False
f = wid.fields
p = wid.params
project = None
package = None
if f.project and f.package:
project = f.project
package = f.package
if p.project and p.package:
project = p.project
package = p.package
if not project or not package:
raise RuntimeError(
"Missing mandatory field or parameter: package, project")
if not f.repourl and not p.repourl:
raise RuntimeError("Missing mandatory field or parameter: repourl")
params = {}
if f.repourl:
params["url"] = f.repourl
if p.repourl:
params["url"] = p.repourl
params["service"], params["repo"] = find_service_repo(params["url"])
if f.branch:
params["branch"] = f.branch
if p.branch:
params["branch"] = p.branch
params["revision"] = ""
if f.revision:
params["revision"] = f.revision
if p.revision:
params["revision"] = p.revision
params["token"] = ""
params["debian"] = ""
params["dumb"] = ""
if f.token:
params["token"] = f.token
if p.token:
params["token"] = p.token
if p.debian:
params["debian"] = p.debian
if f.debian:
params["debian"] = f.debian
if p.dumb:
params["dumb"] = p.dumb
if f.dumb:
params["dumb"] = f.dumb
if "branch" in params and params["branch"].startswith("pkg-"):
if "service" not in params or "repo" not in params:
raise RuntimeError(
"Service/Repo not found in repourl %s " % p.repourl)
service = git_pkg_service
else:
service = tar_git_service
# the simple approach doesn't work with project links
# if self.obs.isNewPackage(project, package):
# self.obs.getCreatePackage(str(project), str(package))
# else:
try:
pkginfo = core.show_files_meta(
self.obs.apiurl, str(project), str(package),
expand=False, meta=True)
if "<entry" not in pkginfo:
# This is a link and it needs branching from the linked project
# so grab the meta and extract the project from the link
print("Found %s as a link in %s" % (package, project))
x = etree.fromstring(
"".join(core.show_project_meta(self.obs.apiurl, project)))
link = x.find('link')
if link is None:
raise Exception(
"Expected a <link> in project %s." % project)
print("Got a link %s" % link)
linked_project = link.get('project')
print("Branching %s to overwrite _service" % package)
core.branch_pkg(self.obs.apiurl, linked_project,
str(package), target_project=str(project))
except Exception as exc:
print("Doing a metatype pkg add because I caught %s" % exc)
print("Creating package %s in project %s" % (package, project))
data = core.metatypes['pkg']['template']
data = StringIO(
data % {
"name": str(package),
"user": self.obs.getUserName()}
).readlines()
u = core.makeurl(
self.obs.apiurl,
['source', str(project), str(package), "_meta"])
x = core.http_PUT(u, data="".join(data))
print("HTTP PUT result of pkg add : %s" % x)
# Set any constraint before we set the service file
constraint_xml = make_constraint(package)
if constraint_xml:
# obs module only exposed the putFile by filepath so
# this is a reimplement to avoid writing a tmpfile
u = core.makeurl(self.obs.apiurl,
['source', project, package, "_constraints"])
core.http_PUT(u, data=constraint_xml)
print "New _constraints file:\n%s" % constraint_xml
else:
print "No _constraints for %s" % package
# Start with an empty XML doc
try: # to get any existing _service file.
# We use expand=0 as otherwise a failed service run won't
# return the _service file
print("Trying to get _service file for %s/%s" % (project, package))
services_xml = self.obs.getFile(
project, package, "_service", expand=0)
except urllib2.HTTPError as e:
print("Exception %s trying to get _service file for %s/%s" %
(e, project, package))
if e.code == 404:
services_xml = empty_service
elif e.code == 400:
# HTTP Error 400: service in progress error
wid.result = True
print("Service in progress, could not get _service file. "
"Not triggering another run.")
return
else:
raise e
services_xml = services_xml.strip() or empty_service
# Replace the matching one:
try:
services = etree.fromstring(services_xml)
except etree.XMLSyntaxError as e:
print(e)
raise
# Create our new service (not services anymore)
new_service_xml = service % params
new_service = etree.fromstring(new_service_xml)
svcname = new_service.find(".").get("name")
old_service = services.find("./service[@name='%s']" % svcname)
if old_service is not None:
services.replace(old_service, new_service)
else:
services.append(new_service)
svc_file = etree.tostring(services, pretty_print=True)
print("New _service file:\n%s" % svc_file)
# And send our new service file
self.obs.setupService(project, package, svc_file)
wid.result = True
def source_file_save(apiurl, project, package, filename, content, comment=None):
comment = message_suffix('updated', comment)
url = makeurl(apiurl, ['source', project, package, filename], {'comment': comment})
http_PUT(url, data=content)
def handle_wi(self, wid):
""" Workitem handling function """
wid.result = False
f = wid.fields
p = wid.params
project = None
package = None
if f.project and f.package:
project = f.project
package = f.package
if p.project and p.package:
project = p.project
package = p.package
if not project or not package:
raise RuntimeError(
"Missing mandatory field or parameter: package, project")
if not f.repourl and not p.repourl:
raise RuntimeError("Missing mandatory field or parameter: repourl")
params = {}
if f.repourl:
params["url"] = f.repourl
if p.repourl:
params["url"] = p.repourl
params["service"], params["repo"] = find_service_repo(params["url"])
if f.branch:
params["branch"] = f.branch
if p.branch:
params["branch"] = p.branch
params["revision"] = ""
if f.revision:
params["revision"] = f.revision
if p.revision:
params["revision"] = p.revision
params["token"] = ""
params["debian"] = ""
params["dumb"] = ""
if f.token:
params["token"] = f.token
if p.token:
params["token"] = p.token
if p.debian:
params["debian"] = p.debian
if f.debian:
params["debian"] = f.debian
if p.dumb:
params["dumb"] = p.dumb
if f.dumb:
params["dumb"] = f.dumb
if "branch" in params and params["branch"].startswith("pkg-"):
if "service" not in params or "repo" not in params:
raise RuntimeError("Service/Repo not found in repourl %s " %
p.repourl)
service = git_pkg_service
else:
service = tar_git_service
# the simple approach doesn't work with project links
# if self.obs.isNewPackage(project, package):
# self.obs.getCreatePackage(str(project), str(package))
# else:
try:
pkginfo = core.show_files_meta(self.obs.apiurl,
str(project),
str(package),
expand=False,
meta=True)
if "<entry" not in pkginfo:
# This is a link and it needs branching from the linked project
# so grab the meta and extract the project from the link
self.log.debug("Found %s as a link in %s" % (package, project))
x = etree.fromstring("".join(
core.show_project_meta(self.obs.apiurl, project)))
link = x.find('link')
if link is None:
raise Exception("Expected a <link> in project %s." %
project)
self.log.debug("Got a link %s" % link)
linked_project = link.get('project')
self.log.debug("Branching %s to overwrite _service" % package)
core.branch_pkg(self.obs.apiurl,
linked_project,
str(package),
target_project=str(project))
except Exception as exc:
self.log.warn("Doing a metatype pkg add because I caught %s" % exc)
self.log.warn("Creating package %s in project %s" %
(package, project))
data = core.metatypes['pkg']['template']
data = StringIO(data % {
"name": str(package),
"user": self.obs.getUserName()
}).readlines()
u = core.makeurl(
self.obs.apiurl,
['source', str(project),
str(package), "_meta"])
x = core.http_PUT(u, data="".join(data))
self.log.debug("HTTP PUT result of pkg add : %s" % x)
# Set any constraint before we set the service file
constraint_xml = self.make_constraint(package)
if constraint_xml:
# obs module only exposed the putFile by filepath so
# this is a reimplement to avoid writing a tmpfile
u = core.makeurl(self.obs.apiurl,
['source', project, package, "_constraints"])
core.http_PUT(u, data=constraint_xml)
self.log.info("New _constraints file:\n%s" % constraint_xml)
else:
self.log.info("No _constraints for %s" % package)
# Start with an empty XML doc
try: # to get any existing _service file.
# We use expand=0 as otherwise a failed service run won't
# return the _service file
self.log.debug("Trying to get _service file for %s/%s" %
(project, package))
services_xml = self.obs.getFile(project,
package,
"_service",
expand=0)
except urllib2.HTTPError as e:
self.log.debug(
"Exception %s trying to get _service file for %s/%s" %
(e, project, package))
if e.code == 404:
services_xml = empty_service
elif e.code == 400:
# HTTP Error 400: service in progress error
wid.result = True
self.log.warn(
"Service in progress, could not get _service file. "
"Not triggering another run.")
return
else:
raise e
services_xml = services_xml.strip() or empty_service
# Replace the matching one:
try:
services = etree.fromstring(services_xml)
except etree.XMLSyntaxError as e:
self.log.exception("Creating services xml failed")
raise
# Create our new service (not services anymore)
new_service_xml = service % params
new_service = etree.fromstring(new_service_xml)
svcname = new_service.find(".").get("name")
old_service = services.find("./service[@name='%s']" % svcname)
if old_service is not None:
services.replace(old_service, new_service)
else:
services.append(new_service)
svc_file = etree.tostring(services, pretty_print=True)
self.log.debug("New _service file:\n%s" % svc_file)
# And send our new service file
self.obs.setupService(project, package, svc_file)
wid.result = True
