def delete(self):
if not hasattr(self, 'id') or not self.id:
raise AgoraException('Cannot delete the task: No ID available')
else:
url = f'{self.BASE_URL}{self.id}/'
response = self.http_client.delete(url, timeout=60)
if response.status_code != 204:
raise AgoraException('Cannot delete the task: ' + response.text)
def progress(self):
url = self.BASE_URL + str(self.id) + '/progress/'
response = self.http_client.get(url)
if response.status_code == 200:
data = response.json()
if 'state' not in data:
raise AgoraException(f'Could not get the task info: {data}')
self.last_progress = data
return self.last_progress
raise AgoraException(f'fail to get progress from ImportPackage {self.id}: {response.status_code}')
def search(cls, search_string, http_client=None):
instance = cls(http_client=http_client)
if not isinstance(search_string, str):
raise AgoraException('The search term must be a string')
url = f'{instance.BASE_URL}search/?q=' + search_string + '&limit=10000000000'
response = http_client.get(url)
if response.status_code == 200:
list = instance.get_list_from_data(response.json())
return list
else:
raise AgoraException(f'Search unsuccessful: {response.text}')
return []
def get_or_create(cls,
username,
password,
email=None,
first_name=None,
last_name=None,
http_client=None):
http_client = get_client(http_client)
url = cls.BASE_URL
data = cls.get_list({'username': username})
for user in data:
if user.username == username:
return user, True
data = {
'username': username,
'password': password,
'email': email,
'first_name': first_name,
'last_name': last_name
}
response = http_client.post(url, data, timeout=60)
if response.status_code == 201:
data = response.json()
if 'id' in data:
new_user = User.from_response(data, http_client)
return new_user, False
raise AgoraException('Could not create the user')
def run(self, input=None, target: BaseModel=None, **kwargs):
if input:
input_dict = self._get_inputs(input)
else:
input_dict = self._get_inputs(kwargs)
self._check_outputs(target)
data = {}
if target:
object_name = target.__class__.__name__.lower()
data['target'] = {'object_id': target.id, 'object_type': object_name}
else:
data['target'] = None
data['inputs'] = input_dict
url = f'{self.BASE_URL}{self.id}/run/'
response = self.http_client.post(url, json=data, timeout=60)
if response.status_code != 200:
raise AgoraException('Cannot run the task: ' + response.text)
else:
taskinfo = json.loads(response.content)
result = TaskInfo.get_list_from_data([taskinfo])
return result[0] if result else None
def _get_inputs(self, arguments):
input_dict = {}
for input in self.inputs:
if not input['key'] in arguments:
raise AgoraException('\n\nThe task input \'' + input['key'] + '\' is unassigned.\nRun the task with the following command:\n\n' + self._get_run_cmd())
argument_name = input['key']
argument = arguments[argument_name]
argument_type = argument.__class__.__name__.lower()
input_type = self._get_type_name(input['type'])
if isinstance(argument, BaseModel):
if argument_type != self._get_type_name(input['type']):
self._raise_input_error(input)
input_dict[argument_name] = {'object_id': argument.id, 'object_type': argument_type}
elif input['type'] < Task.INPUT_TYPE_STRING or input['type'] == Task.INPUT_TYPE_FOLDER:
if not isinstance(argument, int):
self._raise_input_error(input)
input_dict[argument_name] = {'object_id': argument, 'object_type': input_type}
elif input['type'] == Task.INPUT_TYPE_STRING:
if not isinstance(argument, str):
self._raise_input_error(input)
input_dict[argument_name] = argument
elif input['type'] == Task.INPUT_TYPE_INTEGER:
if not isinstance(argument, int):
self._raise_input_error(input)
input_dict[argument_name] = argument
elif input['type'] == Task.INPUT_TYPE_FLOAT:
if not isinstance(argument, float):
self._raise_input_error(input)
input_dict[argument_name] = argument
return input_dict
def delete(self):
url = f'{self.BASE_URL}{self.id}/'
response = self.http_client.delete(url)
if response.status_code == 204:
return True
raise AgoraException('Could not delete FolderItem')
def upload(self, input_files: List[Path], target_folder_id: int = None, json_import_file=None, wait=True,
timeout: int = None, progress=False):
base_url = '/api/v1/import/' + str(self.id) + '/'
url = base_url + 'upload/'
input_files, target_files = self._prepare_paths_to_upload(input_files)
if self._check_zip_option(input_files):
print("Prepare optimized upload")
with tempfile.TemporaryDirectory() as temp_dir:
zip_upload = ZipUploadFiles(input_files, target_files)
input_files, target_files = zip_upload.create_zip(Path(temp_dir))
response = self.http_client.upload(url, input_files, target_files, progress=progress)
else:
self.zip_upload = False
response = self.http_client.upload(url, input_files, target_files, progress=progress)
if self.complete(json_import_file, target_folder_id=target_folder_id):
if wait:
start_time = datetime.datetime.now()
while (datetime.datetime.now() - start_time).seconds < timeout if timeout else True:
data = self.progress()
if data['state'] == 5 or data['state'] == -1:
return data
time.sleep(5)
raise AgoraException(f'Failed to complete upload {self.id}: {response.status_code}')
def get_objects(self):
url = self.BASE_URL + str(self.id) + '/get_objects/'
response = self.http_client.get(url)
if response.status_code == 200:
data = response.json()
return data
raise AgoraException(f'fail to get object from ImportPackage {self.id}: {response.status_code}')
def create(self):
url = self.BASE_URL
response = self.http_client.post(url, json={}, timeout=60)
if response.status_code == 201:
data = response.json()
if 'id' in data:
self._set_values(data)
return self
raise AgoraException("Can't create an Import object")
def save(self):
if not hasattr(self, 'id') or not self.id:
self.create()
else:
url = f'{self.BASE_URL}{self.id}/'
data = self.toDict()
response = self.http_client.put(url, json=data, timeout=60)
if response.status_code != 200:
raise AgoraException('Cannot create a task: ' + response.text)
def get_datasets(self, filters=None):
if filters and not isinstance(filters, dict):
raise AgoraException('The filter must be a dict')
datasets = []
series = self.get_series()
for s in series:
datasets += s.get_datasets(filters)
return datasets
def get_series(self, filters=None):
if filters and not isinstance(filters, dict):
raise AgoraException('The filter must be a dict')
series = []
exams = self.get_exams()
for exam in exams:
series += exam.get_series(filters)
return series
def _get_object(self, id):
url = f'{self.BASE_URL}{id}/'
response = self.http_client.get(url)
if response.status_code == 200:
data = response.json()
return self.__class__.from_response(data,
http_client=self.http_client)
raise AgoraException('Could not get the {0}. HTTP status = {1}'.format(
self.__class__.__name__, response.status_code))
def create_folder(self, name):
url = f'{self.BASE_URL}{self.id}/new/'
post_data = {"name": name}
response = self.http_client.post(url, json=post_data)
if response.status_code == 201:
data = response.json()
if 'content_object' in data:
return Folder.from_response(data['content_object'],
http_client=self.http_client)
raise AgoraException(f'Could not create the folder {name}')
def get_current_user(cls, http_client=None):
url = f'{cls.BASE_URL}current'
http_client = get_client(http_client)
response = http_client.get(url)
if response.status_code == 200:
data = response.json()
return cls.from_response(data, http_client)
raise AgoraException('Could not get the current user')
def get_list(cls, filters=None, http_client=None):
if filters and not isinstance(filters, dict):
raise AgoraException('The filter must be a dict')
instance = cls(http_client=http_client)
filters = filters if filters else {}
if 'limit' not in filters:
filters['limit'] = '10000000000'
url = cls.BASE_URL
return instance._get_object_list(url, filters, cls)
def link_to_folder(self, folder):
from gtagora.models.folder_item import FolderItem
from gtagora.models.folder import Folder
if isinstance(folder, Folder):
folder_id = folder.id
elif isinstance(folder, int):
folder_id = folder
else:
raise AgoraException(
'The input must either be a folder or a folder id')
url = f'{self.BASE_URL}{self.id}/link_to/{folder_id}/'
post_data = {}
response = self.http_client.post(url, post_data)
if response.status_code == 201:
return FolderItem.from_response(response.json(),
http_client=self.http_client)
raise AgoraException('Could not create a link')
def _raise_input_error(self, input):
argument_name = input['key']
argument_type = self._get_type_name(input['type'])
msg = ''
if input['type'] < Task.INPUT_TYPE_STRING or input['type'] == Task.INPUT_TYPE_FOLDER:
msg += '\n\nThe task input \'' + argument_name + '\' must eihter be a ' + argument_type + ' or a ' + argument_type + ' ID'
else:
msg += '\n\nThe task input \'' + argument_name + '\' must be a ' + argument_type
msg += '\n\nRun the task with the following syntax:\n' + self._get_run_cmd()
raise AgoraException(msg)
def complete(self, json_import_file=None, target_folder_id=None):
url = self.BASE_URL + str(self.id) + '/complete/'
post_data = {}
if json_import_file:
post_data.update({'import_file': json_import_file})
if target_folder_id:
post_data.update({'folder': target_folder_id})
response = self.http_client.post(url, json=post_data)
if response.status_code == 204:
return True
raise AgoraException(f'fail to get progress from ImportPackage {self.id}: {response.status_code}')
def set_name(self, name):
url = self.BASE_URL + str(self.id) + '/'
data = {"name": name}
response = self.http_client.put(url, data)
if response.status_code == 200:
data = response.json()
self._set_values(data)
return self
else:
raise AgoraException('Could not set the project name {0}',
response.status_code)
def upload_files(cls,
http_client,
input_files,
target_files,
series_id=None,
exam_id=None,
folder_id=None,
dataset_type=DatasetType.OTHER):
if (not series_id and not folder_id and not exam_id):
raise AgoraException(
'Please specify a SeriesID, ExamID or FolderID')
if isinstance(input_files, str):
Files = []
Files.append(input_files)
else:
Files = input_files
if not target_files:
target_files = [os.path.basename(file) for file in Files]
if len(target_files) < len(Files):
raise AgoraException('TargetFiles list too short.')
for curFile in Files:
if not os.path.isfile(curFile):
raise AgoraException('File does not exist: ' + curFile)
dataset = Dataset(http_client=http_client)
dataset.create(series_id=series_id,
exam_id=exam_id,
folder_id=folder_id,
type=dataset_type)
url = f'/api/v1/dataset/{dataset.id}/upload/'
if http_client.upload(url, input_files, target_files):
return Dataset.get(dataset.id, http_client)
raise AgoraException("Failed to create a new dataset")
def create(self):
data = self.toDict()
data['id'] = None
if 'inputs' in data and data['inputs']:
for input in data['inputs']:
if 'id' in input:
input['id'] = None
response = self.http_client.post(self.BASE_URL, json=data, timeout=60)
if response.status_code != 201:
raise AgoraException('Cannot create a task: ' + response.text)
else:
created_task = json.loads(response.content)
result = self.get_list_from_data([created_task])
return result[0] if result else None
def poll(self, interval=2):
start_time = datetime.datetime.now()
while (datetime.datetime.now() - start_time).seconds < self.TIMEOUT:
task_info = self.get(self.id, http_client=self.http_client)
if task_info.state == 0 or task_info.state == 1:
time.sleep(interval)
continue
elif task_info.state == 2:
return task_info
elif task_info.state == 3:
raise AgoraException(task_info.error)
elif task_info.state.state == 4 or task_info.state.state == 5:
return None
def create(self,
series_id=None,
exam_id=None,
folder_id=None,
type=DatasetType.OTHER):
url = Dataset.BASE_URL
if series_id:
data = {"serie": series_id, "type": type}
elif exam_id:
data = {"exam": exam_id, "type": type}
elif folder_id:
data = {"folder": folder_id, "type": type}
else:
raise AgoraException(
'Please specify a SeriesID, ExamID or FolderID')
response = self.http_client.post(url, json=data, timeout=60)
if response.status_code == 201:
data = response.json()
if 'id' in data:
self._set_values(data)
return self
raise AgoraException('Could not create the dataset')
def is_subfolder_of(self, folder):
if isinstance(folder, Folder):
folder_id = folder.id
elif isinstance(folder, int):
folder_id = folder
else:
raise AgoraException(
'The folder argument must either be a Folder class or a folder ID'
)
breadcrumb = self.get_breadcrumb()
for b in breadcrumb:
if b.object_id == folder_id:
return True
return False
def get_list_from_data(cls, data):
object_list = []
if 'results' in data and 'count' in data:
results = data['results']
if data['count'] == 0:
return object_list
if data['count'] != len(results):
print('Warning: Could not get all series')
for r in results:
object_list.append(cls.from_response(r))
elif isinstance(data, list):
object_list = [cls.from_response(d) for d in data]
return object_list
raise AgoraException(f'Could not get the {cls.__name__} list')
def get_folders(self, parent_folder=None, filters=None):
from gtagora.models.folder import Folder
if filters and not isinstance(filters, dict):
raise AgoraException('The filter must be a dict')
url = f'{self.BASE_URL}{self.id}/folders/?limit=10000000000'
folders = self._get_object_list(url, filters, Folder)
if parent_folder:
filtered_folders = []
for f in folders:
if f.is_subfolder_of(parent_folder):
filtered_folders.append(f)
folders = filtered_folders if filtered_folders else None
return folders
def upload(self, input_files, target_files=None):
if target_files and len(input_files) != len(target_files):
raise AgoraException(
"The Inputfiles and TargetFiles must have the same length")
if isinstance(input_files, str):
files = [input_files]
else:
files = input_files
datasets = []
for index, curFile in enumerate(files):
cur_target_file = None
if target_files:
cur_target_file = target_files[index]
datasets.append(
Dataset.upload_files(self.http_client,
curFile,
cur_target_file,
series_id=self.id))
return datasets
def is_in_folder(self, object: BaseModel, folder):
from gtagora.models.folder import Folder
folder_id = None
if isinstance(folder, Folder):
folder_id = folder.id
elif isinstance(folder, int):
folder_id = folder
else:
raise AgoraException(
'The folder argument must either be a Folder class or a folder ID'
)
folders = object.get_folders()
for f in folders:
breadcrumb = f.get_breadcrumb()
for b in breadcrumb:
if b.object_id == folder_id:
return f
return None
