def upload(self, login, package_name, release, basename, fd, distribution_type,
description='', md5=None, size=None, attrs=None, callback=None):
'''
Upload a new distribution to a package release.
:param login: the login of the package owner
:param package_name: the name of the package
:param version: the version string of the release
:param basename: the basename of the distribution to download
:param fd: a file like object to upload
:param description: (optional) a short description about the file
:param attrs: any extra attributes about the file (eg. build=1, pyversion='2.7', os='osx')
'''
url = '%s/stage/%s/%s/%s/%s' % (self.domain, login, package_name, release, basename)
if attrs is None:
attrs = {}
if not isinstance(attrs, dict):
raise TypeError('argument attrs must be a dictionary')
payload = dict(distribution_type=distribution_type, description=description, attrs=attrs)
data = jencode(payload)
res = self.session.post(url, data=data, verify=True)
self._check_response(res)
obj = res.json()
s3url = obj['s3_url']
s3data = obj['s3form_data']
if md5 is None:
_hexmd5, b64md5, size = compute_hash(fd, size=size)
elif size is None:
spos = fd.tell()
fd.seek(0, os.SEEK_END)
size = fd.tell() - spos
fd.seek(spos)
s3data['Content-Length'] = size
s3data['Content-MD5'] = b64md5
data_stream, headers = stream_multipart(s3data, files={'file':(basename, fd)},
callback=callback)
s3res = requests.post(s3url, data=data_stream, verify=True, timeout=10 * 60 * 60, headers=headers)
if s3res.status_code != 201:
print s3res.text
print
print
raise BinstarError('Error uploading to s3', s3res.status_code)
url = '%s/commit/%s/%s/%s/%s' % (self.domain, login, package_name, release, basename)
payload = dict(dist_id=obj['dist_id'])
data = jencode(payload)
res = self.session.post(url, data=data, verify=True)
self._check_response(res)
return res.json()
def download(self, project, project_dir=None, parent_dir=None):
"""Download project archive and extract."""
if '/' in project:
owner, project_name = project.split('/')
else:
owner = self._username()
project_name = project
if not self._exists(project_name, owner):
raise BinstarError('404')
url = "{}/apps/{}/projects/{}/download".format(self._api.domain, owner,
project_name)
data, headers = binstar_utils.jencode({})
with self._api.session.get(url,
data=data,
headers=headers,
stream=True) as res:
res.raise_for_status()
filename = eval(
re.findall("filename=(.+);",
res.headers["Content-Disposition"])[0])
if parent_dir:
filename = os.path.join(parent_dir, filename)
print('Downloading {}'.format(project))
print('.', end='')
with open(filename, 'wb') as f:
for chunk in res.iter_content(chunk_size=4096):
if chunk:
print('.', end='')
f.write(chunk)
print()
self._check_response(res)
return os.path.abspath(filename)
def submit_for_build(self, username, package, fd, instructions,
test_only=False, callback=None):
url = '%s/build/%s/%s/stage' % (self.domain, username, package)
data = jencode(instructions=instructions, test_only=test_only)
res = self.session.post(url, data=data)
self._check_response(res)
obj = res.json()
s3url = obj['s3_url']
s3data = obj['s3form_data']
_hexmd5, b64md5, size = compute_hash(fd)
s3data['Content-Length'] = size
s3data['Content-MD5'] = b64md5
data_stream, headers = stream_multipart(s3data, files={'file':(obj['basename'], fd)},
callback=callback)
s3res = requests.post(s3url, data=data_stream, verify=True, timeout=10 * 60 * 60, headers=headers)
if s3res.status_code != 201:
raise BinstarError('Error uploading to s3', s3res.status_code)
url = '%s/build/%s/%s/commit/%s' % (self.domain, username, package, obj['build_id'])
res = self.session.post(url, verify=True)
self._check_response(res, [201])
return obj['build_no']
def stage(self):
url = "{}/apps/{}/projects/{}/stage".format(
self.domain, self.username, self.project.name)
data, headers = jencode(self.project.to_stage())
res = self.session.post(url, data=data, headers=headers)
self._check_response(res)
return res
def submit_for_build(self, username, package, fd, instructions,
test_only=False, channels=None, queue=None, queue_tags=None, callback=None):
url = '%s/build/%s/%s/stage' % (self.domain, username, package)
data, headers = jencode(instructions=instructions, test_only=test_only, channels=channels, queue_name=queue, queue_tags=queue_tags)
res = self.session.post(url, data=data, headers=headers)
self._check_response(res)
obj = res.json()
s3url = obj['post_url']
s3data = obj['form_data']
_hexmd5, b64md5, size = compute_hash(fd)
s3data['Content-Length'] = size
s3data['Content-MD5'] = b64md5
data_stream, headers = stream_multipart(s3data, files={'file':(obj['basename'], fd)},
callback=callback)
s3res = requests.post(s3url, data=data_stream, verify=True, timeout=10 * 60 * 60, headers=headers)
if s3res.status_code != 201:
raise BinstarError('Error uploading build', s3res.status_code)
url = '%s/build/%s/%s/commit/%s' % (self.domain, username, package, obj['build_id'])
res = self.session.post(url, verify=True)
self._check_response(res, [201])
return obj
def commit(self, project_name, dist_id):
url = "{}/apps/{}/projects/{}/commit/{}".format(self._api.domain, self._username(), project_name, dist_id)
data, headers = binstar_utils.jencode({})
res = self._api.session.post(url, data=data, headers=headers)
self._check_response(res)
return res
def add_package(self, login, package_name,
summary=None,
license=None,
public=True,
publish=True,
license_url=None,
attrs=None):
'''
Add a new package to a users account
:param login: the login of the package owner
:param package_name: the name of the package to be created
:param package_type: A type identifyer for the package (eg. 'pypi' or 'conda', etc.)
:param summary: A short summary about the package
:param license: the name of the package license
:param license_url: the url of the package license
:param public: if true then the package will be hosted publicly
:param attrs: A dictionary of extra attributes for this package
:param host_publicly: TODO: describe
'''
url = '%s/package/%s/%s' % (self.domain, login, package_name)
attrs = attrs or {}
attrs['summary'] = summary
attrs['license'] = {'name':license, 'url':license_url}
payload = dict(public=bool(public),
publish=bool(publish),
public_attrs=dict(attrs or {})
)
data = jencode(payload)
res = self.session.post(url, verify=True, data=data)
self._check_response(res)
return res.json()
def submit_for_url_build(self,
username,
package,
instructions,
test_only=False,
callback=None,
channels=None,
queue=None,
queue_tags=None,
sub_dir='',
filter_platform=None):
# /build/<owner_login>/<package_name>/submit-git-url
url = '%s/build/%s/%s/submit-git-url' % (self.domain, username,
package)
data, headers = jencode(instructions=instructions,
test_only=test_only,
channels=channels,
sub_dir=sub_dir,
queue_name=queue,
queue_tags=queue_tags,
filter_platform=filter_platform)
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [201])
obj = res.json()
return obj
def upload_worker_stats(self, username, queue_name, worker_id):
url = '%s/build-worker/%s/%s/%s/worker-stats' % (self.domain, username,
queue_name, worker_id)
data, headers = jencode(worker_stats=worker_stats())
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [201])
return res.json()
def copy(self,
owner,
package,
version,
basename=None,
to_owner=None,
from_label='main',
to_label='main',
replace=False,
update=False):
copy_path = "/".join((owner, package, version, basename or ''))
url = '{}/copy/package/{}'.format(self.domain, copy_path)
payload = dict(to_owner=to_owner,
from_channel=from_label,
to_channel=to_label)
data, headers = jencode(payload)
if replace:
res = self.session.put(url, data=data, headers=headers)
elif update:
res = self.session.patch(url, data=data, headers=headers)
else:
res = self.session.post(url, data=data, headers=headers)
try:
self._check_response(res)
except Conflict:
raise Conflict(
'File conflict while copying! Try to use --replace or --update options for force copying'
)
return res.json()
def commit(self, project_name, dist_id):
url = "{}/apps/{}/projects/{}/commit/{}".format(self._api.domain, self._username(), project_name, dist_id)
data, headers = binstar_utils.jencode({})
res = self._api.session.post(url, data=data, headers=headers)
self._check_response(res)
return res
def collection_pull(self, from_org, from_name,
to_org, to_name):
url = '%s/collections/%s/%s/pull' % (self.domain, to_org, to_name)
payload = {'from_owner': from_org, 'from_name': from_name}
data = jencode(payload)
res = self.session.patch(url, data=data, verify=True)
self._check_response(res, [201])
def collection_clone(self, from_org, from_name,
to_org, to_name):
url = '%s/collections/%s/%s' % (self.domain, to_org, to_name)
payload = dict(clone={'owner': from_org, 'collection': from_name})
data = jencode(payload)
res = self.session.post(url, data=data, verify=True)
self._check_response(res, [201])
def register_worker(self, username, queue_name, platform, hostname, dist):
url = '%s/build-worker/%s/%s' % (self.domain, username, queue_name)
data, headers = jencode(platform=platform, hostname=hostname, dist=dist,
binstar_version=binstar_client.__version__,
binstar_build_version=binstar_build_client.__version__)
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [200])
return res.json()['worker_id']
def add_build_queue(self, username, queuename):
url = '%s/build-queues/%s/%s' % (self.domain, username, queuename)
data, headers = jencode()
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [201])
return
def commit(self, revision_id):
url = "{}/apps/{}/projects/{}/commit/{}".format(
self.domain, self.username,
self.project.name, revision_id
)
data, headers = jencode({})
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [201])
return res
def create(self, project_info):
url = "{}/apps/{}/projects".format(self._api.domain, self._username())
json = {'name': project_info['name'],
'access': 'public',
'profile': {'description': project_info['description']}}
data, headers = binstar_utils.jencode(json)
res = self._api.session.post(url, data=data, headers=headers)
self._check_response(res)
return res
def add_build_queue(self, username, queuename):
url = '%s/build-queues/%s/%s' % (self.domain, username, queuename)
data, headers = jencode()
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [201])
return
def create(self, project_info):
url = "{}/apps/{}/projects".format(self._api.domain, self._username())
json = {'name': project_info['name'],
'access': 'public',
'profile': {'description': project_info['description']}}
data, headers = binstar_utils.jencode(json)
res = self._api.session.post(url, data=data, headers=headers)
self._check_response(res)
return res
def copy(self, owner, package, version, basename=None, to_owner=None, from_channel="main", to_channel="main"):
url = "%s/copy/package/%s/%s/%s" % (self.domain, owner, package, version)
if basename:
url += "/%s" % basename
payload = dict(to_owner=to_owner, from_channel=from_channel, to_channel=to_channel)
data, headers = jencode(payload)
res = self.session.post(url, data=data, headers=headers)
self._check_response(res)
return res.json()
def add_group(self, org, name, perms='read'):
url = '%s/group/%s/%s' % (self.domain, org, name)
payload = dict(perms=perms)
data, headers = jencode(payload)
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [204])
return
def update_collection(self, org, name, public=None, description=None):
'''
update a collection
'''
url = '%s/collections/%s/%s' % (self.domain, org, name)
payload = dict(public=public, description=description)
data = jencode(payload)
res = self.session.patch(url, data=data, verify=True)
self._check_response(res, [201])
def add_group(self, org, name, perms='read'):
url = '%s/group/%s/%s' % (self.domain, org, name)
payload = dict(perms=perms)
data, headers = jencode(payload)
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [204])
return
def register_worker(self, username, queue_name, platform, hostname, dist):
url = '%s/build-worker/%s/%s' % (self.domain, username, queue_name)
data, headers = jencode(
platform=platform,
hostname=hostname,
dist=dist,
binstar_version=binstar_client.__version__,
binstar_build_version=binstar_build_client.__version__)
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [200])
return res.json()['worker_id']
def copy(self, owner, package, version, basename=None,
to_owner=None, from_channel='main', to_channel='main'):
url = '%s/copy/package/%s/%s/%s' % (self.domain, owner, package, version)
if basename:
url += '/%s' % basename
payload = dict(to_owner=to_owner, from_channel=from_channel, to_channel=to_channel)
data = jencode(payload)
res = self.session.post(url, data=data)
self._check_response(res)
return res.json()
def stage(self, project_info, archive_filename, uploaded_basename):
url = "{}/apps/{}/projects/{}/stage".format(self._api.domain, self._username(), project_info['name'])
config = project_info.copy()
config['size'] = os.path.getsize(archive_filename)
file_count = self._file_count(archive_filename)
if file_count is not None:
config['num_of_files'] = file_count
json = {'basename': uploaded_basename, 'configuration': config}
data, headers = binstar_utils.jencode(json)
res = self._api.session.post(url, data=data, headers=headers)
self._check_response(res)
return res
def stage(self, project_info, archive_filename, uploaded_basename):
url = "{}/apps/{}/projects/{}/stage".format(self._api.domain, self._username(), project_info['name'])
config = project_info.copy()
config['size'] = os.path.getsize(archive_filename)
file_count = self._file_count(archive_filename)
if file_count is not None:
config['num_of_files'] = file_count
json = {'basename': uploaded_basename, 'configuration': config}
data, headers = binstar_utils.jencode(json)
res = self._api.session.post(url, data=data, headers=headers)
self._check_response(res)
return res
def add_ci(self, username, package,
ghowner, ghrepo,
channels, queue=None, sub_dir=None,
branch='master', email=None):
url = '%s/build/%s/%s/ci' % (self.domain, username, package)
data, headers = jencode(ghowner=ghowner, ghrepo=ghrepo,
channels=channels, queue_name=queue, sub_dir=sub_dir,
branch=branch,
email=email,
)
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [201])
def fininsh_build(self,
username,
queue_name,
worker_id,
job_id,
status='success',
failed=False):
url = '%s/build-worker/%s/%s/%s/jobs/%s/finish' % (
self.domain, username, queue_name, worker_id, job_id)
data, headers = jencode(status=status, failed=failed)
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [200])
return res.json()
def collection_remove_packages(self, org, name, owner_package_data=(), owner=None, package=None):
url = '%s/collections/%s/%s/package' % (self.domain, org, name)
if owner and package:
payload = dict(package_owner=owner, package_name=package)
elif owner_package_data:
payload = owner_package_data
else:
raise TypeError('collection_add_packages must spesify argument "owner_package_data"')
data = jencode(payload)
res = self.session.delete(url, data=data, verify=True)
self._check_response(res, [201])
def add_ci(self, username, package,
ghowner, ghrepo,
channels, queue=None, sub_dir=None,
branch='master', email=None):
url = '%s/build/%s/%s/ci' % (self.domain, username, package)
data, headers = jencode(ghowner=ghowner, ghrepo=ghrepo,
channels=channels, queue_name=queue, sub_dir=sub_dir,
branch=branch,
email=email,
)
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [201])
def trigger_build(self, username, package, channels=None,
queue_name=None, branch=None, test_only=False,
filter_platform=None):
url = '%s/build/%s/%s/trigger' % (self.domain, username, package)
data, headers = jencode(channels=channels, queue_name=queue_name,
branch=branch, test_only=test_only,
filter_platform=filter_platform)
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [201])
obj = res.json()
return obj['build_no']
def add_collection(self, org, name, public=True, description=''):
'''
add a collection to an organization
:param org: the organization username
:param name: the name of the collection to create
:param public: make this collection public
:param description: describe the collection
'''
url = '%s/collections/%s/%s' % (self.domain, org, name)
payload = dict(public=public, description=description)
data = jencode(payload)
res = self.session.post(url, data=data, verify=True)
self._check_response(res, [201])
def add_channel(self, channel, owner, package=None, version=None, filename=None):
'''
Add a channel to the specified files
:param channel: channel to add
:param owner: The user to add the channel to (all files of all packages for this user)
:param package: The package to add the channel to (all files in this package)
:param version: The version to add the channel to (all files in this version of the package)
:param filename: The exact file to add the channel to
'''
url = '%s/channels/%s/%s' % (self.domain, owner, channel)
data = jencode(package=package, version=version, basename=filename)
res = self.session.post(url, data=data)
self._check_response(res, [201])
def remove_channel(self, channel, owner, package=None, version=None, filename=None):
'''
Remove a channel from the specified files
:param channel: channel to remove
:param owner: The user to remove the channel from (all files of all packages for this user)
:param package: The package to remove the channel from (all files in this package)
:param version: The version to remove the channel to (all files in this version of the package)
:param filename: The exact file to remove the channel from
'''
url = '%s/channels/%s/%s' % (self.domain, owner, channel)
data, headers = jencode(package=package, version=version, basename=filename)
res = self.session.delete(url, data=data, headers=headers)
self._check_response(res, [201])
def trigger_build(self, username, package, channels=None,
queue_name=None, queue_tags=None, branch=None, test_only=False,
filter_platform=None):
url = '%s/build/%s/%s/trigger' % (self.domain, username, package)
data, headers = jencode(channels=channels,
queue_name=queue_name,
queue_tags=queue_tags,
branch=branch,
test_only=test_only,
filter_platform=filter_platform)
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [201])
obj = res.json()
return obj['build_no']
def submit_for_url_build(self, username, package, instructions,
test_only=False, callback=None,
channels=None, queue=None, queue_tags=None, sub_dir='',
filter_platform=None):
# /build/<owner_login>/<package_name>/submit-git-url
url = '%s/build/%s/%s/submit-git-url' % (self.domain, username, package)
data, headers = jencode(instructions=instructions, test_only=test_only,
channels=channels, sub_dir=sub_dir,
queue_name=queue, queue_tags=queue_tags,
filter_platform=filter_platform)
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [201])
obj = res.json()
return obj
def add_release(self, login, package_name, version, requirements, announce, description):
'''
Add a new release to a package.
:param login: the login of the package owner
:param package_name: the name of the package
:param version: the version string of the release
:param requirements: A dict of requirements TODO: describe
:param announce: An announcement that will be posted to all package watchers
:param description: A long description about the package
'''
url = '%s/release/%s/%s/%s' % (self.domain, login, package_name, version)
payload = {'requirements':requirements, 'announce':announce, 'description':description}
data = jencode(payload)
res = self.session.post(url, data=data, verify=True)
self._check_response(res)
return res.json()
def remove_channel(self,
channel,
owner,
package=None,
version=None,
filename=None):
'''
Remove a channel from the specified files
:param channel: channel to remove
:param owner: The user to remove the channel from (all files of all packages for this user)
:param package: The package to remove the channel from (all files in this package)
:param version: The version to remove the channel to (all files in this version of the package)
:param filename: The exact file to remove the channel from
'''
url = '%s/channels/%s/%s' % (self.domain, channel, owner)
data, headers = jencode(package=package,
version=version,
basename=filename)
res = self.session.delete(url, data=data, headers=headers)
self._check_response(res, [201])
def download(self, project, project_dir=None, parent_dir=None):
"""Download project archive and extract."""
if '/' in project:
owner, project_name = project.split('/')
else:
owner = self._username()
project_name = project
if not self._exists(project_name, owner):
raise BinstarError('404')
url = "{}/apps/{}/projects/{}/download".format(self._api.domain, owner,
project_name)
data, headers = binstar_utils.jencode({})
with self._api.session.get(url,
data=data,
headers=headers,
stream=True) as res:
res.raise_for_status()
filename = eval(
re.findall("filename=(.+);",
res.headers["Content-Disposition"])[0])
if parent_dir:
filename = os.path.join(parent_dir, filename)
progress = tqdm(
unit='KiB',
total=int(res.headers.get('Content-Length', None)) / 1024,
unit_scale=True,
desc='Download')
with open(filename, 'wb') as f:
for chunk in res.iter_content(chunk_size=1024):
if chunk:
progress.update(len(chunk) / 1024)
f.write(chunk)
progress.close()
self._check_response(res)
return os.path.abspath(filename)
def upload_worker_stats(self, username, queue_name, worker_id):
url = '%s/build-worker/%s/%s/%s/worker-stats' % (self.domain, username, queue_name, worker_id)
data, headers = jencode(worker_stats=worker_stats())
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [201])
return res.json()
def projects(self):
url = "{}/apps/{}/projects".format(self.domain, self.username)
data, headers = jencode(self.project.to_project_creation())
return self.session.get(url, data=data, headers=headers)
def set_keyfile(self, username, package, filename, content):
url = '%s/build/%s/%s/keyfile' % (self.domain, username, package)
data = jencode(filename=filename, content=content)
res = self.session.post(url, data=data)
self._check_response(res, [201])
def set_keyfile(self, username, package, filename, content):
url = '%s/build/%s/%s/keyfile' % (self.domain, username, package)
data, headers = jencode(filename=filename, content=content)
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [201])
def fininsh_build(self, username, queue_name, worker_id, job_id, status='success', failed=False):
url = '%s/build-worker/%s/%s/%s/jobs/%s/finish' % (self.domain, username, queue_name, worker_id, job_id)
data, headers = jencode(status=status, failed=failed)
res = self.session.post(url, data=data, headers=headers)
self._check_response(res, [200])
return res.json()