def create_folder(cls, name, parent=None, project=None,
api=None):
"""Create a new folder
:param name: Folder name
:param parent: Parent folder
:param project: Project to create folder in
:param api: Api instance
:return: New folder
"""
api = api or cls._API
data = {
'name': name,
'type': cls.FOLDER_TYPE
}
if not parent and not project:
raise SbgError('Parent or project must be provided')
if parent and project:
raise SbgError(
'Providing both "parent" and "project" is not allowed'
)
if parent:
data['parent'] = Transform.to_file(file_=parent)
if project:
data['project'] = Transform.to_project(project=project)
response = api.post(url=cls._URL['create_folder'], data=data).json()
return cls(api=api, **response)
def _download_part(path, session, url, timeout, start_byte, end_byte):
"""
Downloads a single part.
:param path: File path.
:param session: Requests session.
:param url: Url of the resource.
:param timeout: Session timeout.
:param start_byte: Start byte of the part.
:param end_byte: End byte of the part.
:return:
"""
try:
fp = os.open(path, os.O_CREAT | os.O_WRONLY)
except IOError:
raise SbgError('Unable to open file %s' % path)
# Prepare range headers.
headers = {}
if end_byte is not None:
headers['Range'] = 'bytes=%d-%d' % (int(start_byte), int(end_byte))
try:
response = session.get(url=url,
headers=headers,
timeout=timeout,
stream=True)
response.raise_for_status()
part_size = response.headers.get('Content-Length')
os.lseek(fp, start_byte, os.SEEK_SET)
for part in response.iter_content(32 * PartSize.KB):
os.write(fp, part)
os.close(fp)
return Part(start=start_byte, size=float(part_size))
except (requests.HTTPError, requests.RequestException) as e:
raise SbgError(f'Failed to download file. Response: {e}')
def list_files(self, offset=None, limit=None, api=None, cont_token=None):
"""List files in a folder
:param api: Api instance
:param offset: Pagination offset
:param limit: Pagination limit
:param cont_token: Pagination continuation token
:return: List of files
"""
if cont_token and offset:
raise SbgError(
'Offset and continuation token parameters'
'are mutually exclusive.'
)
api = api or self._API
if not self.is_folder():
raise SbgError('{name} is not a folder'.format(name=self.name))
url = self._URL[
'scroll_folder' if cont_token else 'list_folder'
].format(id=self.id)
return super(File, self.__class__)._query(
api=api, url=url, token=cont_token, offset=offset,
limit=limit, fields='_all'
)
def _download_chunk(file_path, session, url, retry, timeout, start_byte,
end_byte):
try:
fp = os.open(file_path, os.O_CREAT | os.O_WRONLY)
except IOError:
raise SbgError('Unable to open file %s' % file_path)
headers = {}
if end_byte is not None:
headers['Range'] = 'bytes=%d-%d' % (int(start_byte), int(end_byte))
for retry in range(retry):
try:
response = session.get(url,
headers=headers,
timeout=timeout,
stream=True)
chunk_size = response.headers.get('Content-Length')
os.lseek(fp, start_byte, os.SEEK_SET)
for chunk in response.iter_content(32 * PartSize.KB):
os.write(fp, chunk)
os.close(fp)
except requests.RequestException:
time.sleep(2**retry)
continue
else:
return Chunk(start=start_byte, size=float(chunk_size))
else:
os.close(fp)
error = SbgError('Failed to download file after %s attempts.' % retry)
raise error
def wrapper(*args, **kwargs):
try:
response = func(*args, **kwargs)
status_code = response.status_code
if status_code in range(200, 204):
return response
if status_code == 204:
return
data = response.json()
e = {
400: BadRequest,
401: Unauthorized,
403: Forbidden,
404: NotFound,
405: MethodNotAllowed,
408: RequestTimeout,
409: Conflict,
429: TooManyRequests,
}.get(status_code, SbgError)()
if 'message' in data:
e.message = data['message']
if 'code' in data:
e.code = data['code']
if 'status' in data:
e.status = data['status']
if 'more_info' in data:
e.more_info = data['more_info']
raise e
except requests.RequestException as e:
raise SbgError(message=str(e))
except ValueError as e:
raise SbgError(message=str(e))
def reload(self):
"""
Refreshes the resource with the data from the server.
"""
try:
if hasattr(self, 'href'):
data = self._api.get(self.href, append_base=False).json()
resource = self.__class__(api=self._api, **data)
elif hasattr(self, 'id') and hasattr(self, '_URL') and \
'get' in self._URL:
data = self._api.get(
self._URL['get'].format(id=self.id)).json()
resource = self.__class__(api=self._api, **data)
else:
raise SbgError(
'Resource can not be refreshed, "id" property not set or '
'retrieval for this resource is not available.')
query = {'id': self.id} if hasattr(self, 'id') else {}
extra = {'resource': self.__class__.__name__, 'query': query}
logger.info('Reloading {} resource.'.format(self), extra=extra)
except Exception as e:
raise SbgError(
'Resource can not be refreshed due to an error: {}'.format(
six.text_type(e)))
self._data = resource._data
self._dirty = resource._dirty
self.update_old()
return self
def run_tasks(api):
logger.info('Running tasks!')
running_tasks = list(
api.tasks.query(project=project, limit=100, status='RUNNING').all())
queued_tasks = list(
api.tasks.query(project=project, limit=100, status='QUEUED').all())
if len(running_tasks) + len(queued_tasks) >= max_task_number:
logger.info("Maximum number of active tasks reached!")
raise SbgError(
'Unable to run! You already have {active} active tasks. '
'Please try later!'.format(active=len(running_tasks) +
len(queued_tasks)))
draft_tasks = list(
api.tasks.query(project=project, limit=100, status='DRAFT').all())
if len(draft_tasks) == 0:
print('No draft tasks left to be run!')
return
executable_tasks = draft_tasks[0:max_task_number - len(running_tasks)]
for task in executable_tasks:
try:
task.run()
except SbgError as e:
logger.error("Task was not started! Error happened ", exc_info=e)
raise SbgError('Task was not started! Error happened')
if task.status == 'DRAFT':
logger.error("Task was not started! Task state is DRAFT!")
raise SbgError("Task was not started! Task state is DRAFT!")
def _validate_project_parent(self, parent, project):
if not project and not parent:
raise SbgError('Project or parent identifier is required.')
if project and parent:
raise SbgError(
'Project and parent identifiers are mutually exclusive.'
)
def reload(self):
"""
Refreshes the resource with the data from the server.
"""
try:
if hasattr(self, 'href'):
data = self._api.get(self.href, append_base=False).json()
resource = self.__class__(api=self._api, **data)
elif hasattr(self, 'id') and hasattr(self, '_URL') and \
'get' in self._URL:
data = self._api.get(
self._URL['get'].format(id=self.id)).json()
resource = self.__class__(api=self._api, **data)
else:
raise SbgError('Resource can not be refreshed!')
query = {'id': self.id} if hasattr(self, 'id') else {}
extra = {'resource': self.__class__.__name__, 'query': query}
logger.info('Reloading {} resource.'.format(self), extra=extra)
except Exception:
raise SbgError('Resource can not be refreshed!')
self._data = resource._data
self._dirty = resource._dirty
self._old = copy.deepcopy(self._data.data)
return self
def upload(cls, path, project=None, parent=None, file_name=None,
overwrite=False, retry=5, timeout=60, part_size=None, wait=True,
api=None):
"""
Uploads a file using multipart upload and returns an upload handle
if the wait parameter is set to False. If wait is set to True it
will block until the upload is completed.
:param path: File path on local disc.
:param project: Project identifier
:param parent: Parent folder identifier
:param file_name: Optional file name.
:param overwrite: If true will overwrite the file on the server.
:param retry: Number of retries if error occurs during upload.
:param timeout: Timeout for http requests.
:param part_size: Part size in bytes.
:param wait: If true will wait for upload to complete.
:param api: Api instance.
"""
api = api or cls._API
extra = {'resource': cls.__name__, 'query': {
'path': path,
'project': project,
'file_name': file_name,
'overwrite': overwrite,
'retry': retry,
'timeout': timeout,
'part_size': part_size,
'wait': wait,
}}
logger.info('Uploading file', extra=extra)
if not project and not parent:
raise SbgError('A project or parent identifier is required.')
if project and parent:
raise SbgError(
'Project and parent identifiers are mutually exclusive.'
)
if project:
project = Transform.to_project(project)
if parent:
parent = Transform.to_file(parent)
upload = Upload(
file_path=path, project=project, parent=parent,
file_name=file_name, overwrite=overwrite, retry_count=retry,
timeout=timeout, part_size=part_size, api=api
)
if wait:
upload.start()
upload.wait()
return upload
else:
return upload
def to_tags(tags):
if not isinstance(tags, list):
raise SbgError('Tags argument must be a list.')
tag_list = []
for tag in tags:
if "," in tag:
raise SbgError('Tags must not contain comma character.')
tag_list.append(tag)
return tag_list
def __init__(self,
url=None,
file_path=None,
retry=5,
timeout=10,
chunk_size=None,
api=None):
"""
File downloader.
:param url: URL of the file.
:param file_path: Local file path.
:param retry: Retry count.
:param timeout: Connection timeout in seconds.
:param chunk_size: Size of the chunks in bytes.
:param api: sbApi instance.
"""
threading.Thread.__init__(self)
if not url:
raise SbgError('Url must be supplied.')
if not file_path:
raise SbgError('File path must be supplied.')
if api is None:
raise SbgError('Api instance not supplied.')
if chunk_size and chunk_size < PartSize.MINIMUM_PART_SIZE:
raise SbgError(
message='Chunk size is too small! Minimum chunk size is %s' %
PartSize.MINIMUM_PART_SIZE)
self._session = requests.Session()
self.url = url
self._file_path = file_path
self._temp_file = self._file_path + '.' + hashlib.sha1(
self._file_path.encode('utf-8')).hexdigest()[:10]
self._retry = retry
self._timeout = timeout
if chunk_size:
self._chunk_size = chunk_size
else:
self._chunk_size = PartSize.DOWNLOAD_MINIMUM_PART_SIZE
self._api = api
self._bytes_done = 0
self._running = threading.Event()
self._callback = None
self._errorback = None
self._progress_callback = None
self._time_started = 0
try:
self._file_size = self._get_file_size()
except SbgError as error:
if self._errorback:
self._errorback(error)
else:
raise error
self._status = TransferState.NOT_INITIALIZED
self._stop_signal = False
def bulk_submit(cls, imports, api=None):
"""
Submit imports in bulk
:param imports: List of dicts describing a wanted import.
:param api: Api instance.
:return: List of ImportBulkRecord objects.
"""
if not imports:
raise SbgError('Imports are required')
api = api or cls._API
items = []
for import_ in imports:
project = import_.get('project')
parent = import_.get('parent')
if project and parent:
raise SbgError(
'Project and parent identifiers are mutually exclusive')
elif project:
destination = {'project': Transform.to_project(project)}
elif parent:
destination = {'parent': Transform.to_file(parent)}
else:
raise SbgError('Project or parent identifier is required.')
volume = Transform.to_volume(import_.get('volume'))
location = Transform.to_location(import_.get('location'))
name = import_.get('name', None)
overwrite = import_.get('overwrite', False)
autorename = import_.get('autorename', None)
preserve_folder_structure = import_.get(
'preserve_folder_structure', None)
if name:
destination['name'] = name
import_config = {
'source': {
'volume': volume,
'location': location
},
'destination': destination,
'overwrite': overwrite,
}
if autorename is not None:
import_config['autorename'] = autorename
if preserve_folder_structure is not None:
import_config['preserve_folder_structure'] = (
preserve_folder_structure)
items.append(import_config)
data = {'items': items}
response = api.post(url=cls._URL['bulk_create'], data=data)
return ImportBulkRecord.parse_records(response=response, api=api)
def to_async_job(async_job):
from sevenbridges.models.async_jobs import AsyncJob
if not async_job:
raise SbgError('Async job is required!')
if isinstance(async_job, AsyncJob):
return async_job.id
if isinstance(async_job, six.string_types):
return async_job
else:
raise SbgError('Invalid async job parameter!')
def to_automation_package(package):
from sevenbridges.models.automation import AutomationPackage
if not package:
raise SbgError('Automation package is required!')
if isinstance(package, AutomationPackage):
return package.id
if isinstance(package, six.string_types):
return package
else:
raise SbgError('Invalid automation package parameter!')
def to_automation_member(member):
from sevenbridges.models.automation import AutomationMember
if not member:
raise SbgError('Automation member is required!')
if isinstance(member, AutomationMember):
return member.username
if isinstance(member, six.string_types):
return member
else:
raise SbgError('Invalid automation member parameter!')
def to_automation(automation):
from sevenbridges.models.automation import Automation
if not automation:
raise SbgError('Automation is required!')
if isinstance(automation, Automation):
return automation.id
if isinstance(automation, six.string_types):
return automation
else:
raise SbgError('Invalid automation parameter!')
def to_member(member):
from sevenbridges.models.member import Member
if not member:
raise SbgError('Member is required!')
if isinstance(member, Member):
return member.username
if isinstance(member, six.string_types):
return member
else:
raise SbgError('Invalid member parameter!')
def to_dataset(dataset):
from sevenbridges.models.dataset import Dataset
if not dataset:
raise SbgError('Dataset is required!')
if isinstance(dataset, Dataset):
return dataset.id
if isinstance(dataset, six.string_types):
return dataset
else:
raise SbgError('Invalid dataset parameter!')
def to_team(team):
from sevenbridges.models.team import Team
if not team:
raise SbgError('Team is required!')
elif isinstance(team, Team):
return team.id
elif isinstance(team, six.string_types):
return team
else:
raise SbgError('Invalid team parameter!')
def to_project(project):
from sevenbridges.models.project import Project
if not project:
raise SbgError('Project is required!')
elif isinstance(project, Project):
return project.id
elif isinstance(project, six.string_types):
return project
else:
raise SbgError('Invalid project parameter!')
def to_volume(volume):
from sevenbridges.models.volume import Volume
if not volume:
raise SbgError('Volume is required!')
elif isinstance(volume, Volume):
return volume.id
elif isinstance(volume, six.string_types):
return volume
else:
raise SbgError('Invalid volume parameter!')
def to_file(file_):
from sevenbridges.models.file import File
if not file_:
raise SbgError('File is required!')
elif isinstance(file_, File):
return file_.id
elif isinstance(file_, six.string_types):
return file_
else:
raise SbgError('Invalid file parameter!')
def to_app(app):
from sevenbridges.models.app import App
if not app:
raise SbgError('App is required!')
elif isinstance(app, App):
return app.id
elif isinstance(app, six.string_types):
return app
else:
raise SbgError('Invalid app parameter!')
def to_task(task):
from sevenbridges.models.task import Task
if not task:
raise SbgError('Task is required!')
elif isinstance(task, Task):
return task.id
elif isinstance(task, six.string_types):
return task
else:
raise SbgError('Invalid task parameter!')
def to_resource(resource):
from sevenbridges.meta.resource import Resource
if not resource:
raise SbgError('Invalid id value!')
elif isinstance(resource, Resource) and hasattr(resource, 'id'):
return resource.id
elif isinstance(resource, six.string_types):
return resource
else:
raise SbgError('Invalid id value!')
def to_marker(marker):
from sevenbridges.models.marker import Marker
if not marker:
raise SbgError('Marker is required!')
elif isinstance(marker, Marker):
return marker.id
elif isinstance(marker, six.string_types):
return marker
else:
raise SbgError('Invalid marker parameter!')
def to_division(division):
from sevenbridges.models.division import Division
if not division:
raise SbgError('Division is required!')
elif isinstance(division, Division):
return division.id
elif isinstance(division, six.string_types):
return division
else:
raise SbgError('Invalid division parameter!')
def to_user(user):
from sevenbridges.models.user import User
if not user:
raise SbgError('User is required!')
elif isinstance(user, User):
return user.username
elif isinstance(user, six.string_types):
return user
else:
raise SbgError('Invalid user parameter!')
def to_billing_group(billing_group):
from sevenbridges.models.billing_group import BillingGroup
if not billing_group:
raise SbgError('Billing group is required!')
elif isinstance(billing_group, BillingGroup):
return billing_group.id
elif isinstance(billing_group, six.string_types):
return billing_group
else:
raise SbgError('Invalid billing group parameter!')
