def wrapper(*args, **kwargs):
new_args = []
for arg in args:
if isinstance(arg, str):
# when a json was supplied, we deserialize it to dict
try:
new_args.append(json.loads(arg))
except:
# regular string from a user
new_args.append(arg)
else:
new_args.append(arg)
if FileId.is_file_id(new_args[-1]):
new_args[-1] = FileId(new_args[-1])
for key, value in kwargs.iteritems():
if isinstance(value, str):
# when a json was supplied, we deserialize it to dict
try:
kwargs[key] = json.loads(value)
except:
pass
if FileId.is_file_id(kwargs[key]):
kwargs[key] = FileId(value)
return func(*tuple(new_args), **kwargs)
def wrapper(*args, **kwargs):
new_args = []
for arg in args:
if isinstance(arg, str):
# when a json was supplied, we deserialize it to dict
try:
new_args.append(json.loads(arg))
except:
# regular string from a user
new_args.append(arg)
else:
new_args.append(arg)
if FileId.is_file_id(new_args[-1]):
new_args[-1] = FileId(new_args[-1])
for key, value in kwargs.iteritems():
if isinstance(value, str):
# when a json was supplied, we deserialize it to dict
try:
kwargs[key] = json.loads(value)
except:
pass
if FileId.is_file_id(kwargs[key]):
kwargs[key] = FileId(value)
return func(*tuple(new_args), **kwargs)
def spec_get(self, package_name, branch=None, commit=None):
'''
Get specfile of a package packaged in Fedora
@param package_name: package name
@param branch: branch (e.g. "f23", ...); if omitted "rawhide" is used
@param commit: fedpkg git commit; if omitted, the latest commit is used
@return: specfile file id
'''
ret = ServiceResult()
# prevent from accessing suspicious files
package_name = os.path.basename(package_name)
if branch is None or branch == "rawhide":
branch = "master"
# we have to ensure that such package/branch/commit exist
with self.get_lock(package_name):
path = self._git_tree_prepare(package_name, branch, commit)
ident = "%s/%s/%s/%s.spec" % (
package_name, branch, self._commit2ident(commit), package_name)
ret.result = FileId.construct(self,
ident,
path=os.path.join(
path, "%s.spec" % package_name))
return ret
def tarball_get(self, upstream_url, commit):
'''
Retrieve a tarball from upstream, try to detect upstream provider
@param upstream_url: an upstream url
@param commit: commit of upstream file
@return: tarball file id
'''
res = ServiceResult()
if upstream_url.startswith('https://github.com'):
m = re.search('https://github.com/([a-z]+)/([a-z-_]+)/?', upstream_url)
if m is None:
raise ValueError("Expected URL in form 'https://github.com/<USER>/<REPO>/, got %s", (upstream_url,))
tarball_url = self._get_github_tarball_url(m.group(1), m.group(2), commit)
filename = self._get_github_file_name(m.group(1), m.group(2), commit)
elif upstream_url.startswith('https://bitbucket.org'):
m = re.search('https://bitbucket.org/([a-z]+)/([a-z-_]+)/?', upstream_url)
if m is None:
raise ValueError("Expected URL in form 'https://bitbucket.org/<USER>/<REPO>/, got %s", (upstream_url,))
tarball_url = self._get_bitbucket_tarball_url(m.group(1), m.group(2), commit)
filename = self._get_bitbucket_file_name(m.group(1), m.group(2), commit)
else:
raise NotImplementedError("Unknown upstream provider %s" % (upstream_url,))
with self.get_lock(filename):
if self.dircache.is_available(filename):
res.result = FileId.construct(self, self.dircache.get_file_path(filename))
else:
res.result = self._download_tarball(tarball_url, filename)
return res
def spec_patch_get(self, package_name, patch_name, branch=None, commit=None):
"""
Get file id of a downstream patch of a package packaged in Fedora
@param package_name: package name
@param patch_name: name of the patch file
@param branch: branch (e.g. "f23", ...); if omitted, "rawhide" is used
@param commit: fedpkg git commit; if omitted, the latest commit is used
@return: file id of the patch
"""
ret = ServiceResult()
ret.result = []
# prevent from accessing suspicious files
package_name = os.path.basename(package_name)
patch_name = os.path.basename(patch_name)
if branch is None or branch == "rawhide":
branch = "master"
with self.get_lock(package_name):
path = self._git_tree_prepare(package_name, branch, commit)
patch_path = os.path.join(path, patch_name)
if not os.path.isfile(patch_path):
raise ValueError(
"There is not patch %s for package %s, branch %s and commit %s"
% (patch_name, package_name, branch, commit)
)
ident = ("%s/%s/%s/%s" % (package_name, branch, self._commit2ident(commit), patch_name),)
ret.result = FileId.construct(self, ident, path=patch_path)
return ret
def _download_tarball(self, tarball_url, filename):
log.debug("Downloading from %s" % (tarball_url,))
response = urllib2.urlopen(tarball_url)
blob = response.read()
h = blob_hash(blob)
self.dircache.store(blob, filename)
return FileId.construct(self, self.dircache.get_file_path(filename), hash_ = h)
def _common_get(self, url, filename):
ret = ServiceResult()
with self.get_lock(filename):
if self.dircache.is_available(filename):
ret.result = FileId.construct(self, self.dircache.get_file_path(filename))
elif remote_exists(url):
log.debug("Downloading from %s" % (url,))
response = urllib2.urlopen(url)
blob = response.read()
h = blob_hash(blob)
self.dircache.store(blob, filename)
ret.result = FileId.construct(self, self.dircache.get_file_path(filename), hash_ = h)
else:
raise KeyError("Desired file '%s' does not exist ( %s )" % (filename, url))
ret.meta['origin'] = url
return ret
def scm_store(self, repo_url, commit=None, branch=None):
'''
Store a SCM repo
@param repo_url: repo URL
@param commit: commit hash; if None, the latest is used
@param branch: branch; if None, "master" is used
@return:
'''
ret = ServiceResult()
if not branch:
branch = "master"
if commit:
commit = commit[:7]
dirname = self._get_dirname(repo_url, commit, branch)
filename = self._get_filename(dirname)
dst_path = self.dircache.get_location(dirname)
with self.get_lock(dirname):
if not self.dircache.is_available(filename):
repo = GitCmd.git_clone_repo(repo_url, dst_path)
repo.git_checkout(branch)
if commit:
repo.git_checkout(commit)
else:
commit = repo.git_rev_parse_head(dst_path)[:7]
# if user did not supplied commit, we have to check it explicitly
filename_old = filename
filename = self._get_filename(
self._get_dirname(repo_url, commit, branch))
# we have to move it so it will be available with specified commit and branch
if filename_old != filename:
shutil.move(filename_old, filename)
if not self.dircache.is_available(filename):
# if user did not supplied commit, we have to pack the repo
self._pack_repo(dirname, filename)
shutil.rmtree(dst_path)
if not self.dircache.is_available(filename):
self.dircache.register(filename)
ret.result = FileId.construct(self,
self.dircache.get_file_path(filename))
ret.meta = {'origin': repo_url}
return ret
def scm_store(self, repo_url, commit=None, branch=None):
'''
Store a SCM repo
@param repo_url: repo URL
@param commit: commit hash; if None, the latest is used
@param branch: branch; if None, "master" is used
@return:
'''
ret = ServiceResult()
if not branch:
branch = "master"
if commit:
commit = commit[:7]
dirname = self._get_dirname(repo_url, commit, branch)
filename = self._get_filename(dirname)
dst_path = self.dircache.get_location(dirname)
with self.get_lock(dirname):
if not self.dircache.is_available(filename):
repo = GitCmd.git_clone_repo(repo_url, dst_path)
repo.git_checkout(branch)
if commit:
repo.git_checkout(commit)
else:
commit = repo.git_rev_parse_head(dst_path)[:7]
# if user did not supplied commit, we have to check it explicitly
filename_old = filename
filename = self._get_filename(self._get_dirname(repo_url, commit, branch))
# we have to move it so it will be available with specified commit and branch
if filename_old != filename:
shutil.move(filename_old, filename)
if not self.dircache.is_available(filename):
# if user did not supplied commit, we have to pack the repo
self._pack_repo(dirname, filename)
shutil.rmtree(dst_path)
if not self.dircache.is_available(filename):
self.dircache.register(filename)
ret.result = FileId.construct(self, self.dircache.get_file_path(filename))
ret.meta = {'origin': repo_url}
return ret
def upload(self, blob):
'''
Upload file to the system
@param blob: a file content to be store
@return: file id
'''
res = ServiceResult()
log.info("uploading")
h = blob_hash(blob)
dst = os.path.join(self.upload_dir, h)
with self.get_lock():
with open(dst, 'wb') as f:
f.write(blob)
creation_time = datetime_parse(time.ctime(os.path.getctime(dst)))
# TODO: remove, use dircache instead
valid_until = creation_time + self.file_lifetime
res.result = FileId.construct(self, dst, h)
return res
def spec_get(self, package_name, branch=None, commit=None):
"""
Get specfile of a package packaged in Fedora
@param package_name: package name
@param branch: branch (e.g. "f23", ...); if omitted "rawhide" is used
@param commit: fedpkg git commit; if omitted, the latest commit is used
@return: specfile file id
"""
ret = ServiceResult()
# prevent from accessing suspicious files
package_name = os.path.basename(package_name)
if branch is None or branch == "rawhide":
branch = "master"
# we have to ensure that such package/branch/commit exist
with self.get_lock(package_name):
path = self._git_tree_prepare(package_name, branch, commit)
ident = "%s/%s/%s/%s.spec" % (package_name, branch, self._commit2ident(commit), package_name)
ret.result = FileId.construct(self, ident, path=os.path.join(path, "%s.spec" % package_name))
return ret
def spec_patch_get(self,
package_name,
patch_name,
branch=None,
commit=None):
'''
Get file id of a downstream patch of a package packaged in Fedora
@param package_name: package name
@param patch_name: name of the patch file
@param branch: branch (e.g. "f23", ...); if omitted, "rawhide" is used
@param commit: fedpkg git commit; if omitted, the latest commit is used
@return: file id of the patch
'''
ret = ServiceResult()
ret.result = []
# prevent from accessing suspicious files
package_name = os.path.basename(package_name)
patch_name = os.path.basename(patch_name)
if branch is None or branch == "rawhide":
branch = "master"
with self.get_lock(package_name):
path = self._git_tree_prepare(package_name, branch, commit)
patch_path = os.path.join(path, patch_name)
if not os.path.isfile(patch_path):
raise ValueError(
"There is not patch %s for package %s, branch %s and commit %s"
% (patch_name, package_name, branch, commit))
ident = "%s/%s/%s/%s" % (package_name, branch,
self._commit2ident(commit), patch_name),
ret.result = FileId.construct(self, ident, path=patch_path)
return ret