def analyze(cls, user_id, project_id, original_id): project = ProjectManager().get_project(project_id, user_id) label_type = project['label_type'] original = OriginalManager().get_original(project_id, original_id, status='uploaded') storage = StorageSerializer().get_storage(project_id, original['storage_id']) storage_config = copy.deepcopy(storage['storage_config']) original_path = StorageSerializer.get_original_path( storage['storage_type'], storage['storage_config'], original['name']) storage_config.update({'path': original_path}) automan_config = cls.__get_automan_config(user_id) automan_config.update({'path': '/projects/' + project_id + '/originals/' + str(original_id) + '/', 'label_type': label_type}) job_config = { 'storage_type': storage['storage_type'], 'storage_config': storage_config, 'automan_config': automan_config } job_config_json = json.dumps(job_config) new_job = Job( job_type='analyzer', user_id=user_id, project_id=project_id, job_config=job_config_json) new_job.save() if original['file_type'] == 'rosbag': job = RosbagAnalyzer(**job_config) job.create(cls.__generate_job_name(new_job.id, 'analyzer')) res = job.run() return res else: raise ValidationError()
def create(self, request): username = request.user user_id = AccountManager.get_id_by_username(username) name = request.data.get('name', None) # create project serializer = ProjectSerializer(data={ 'name': name, 'description': request.data.get('description', None), 'label_type': request.data.get('label_type', None), 'owner_id': user_id }) if not serializer.is_valid(): raise ValidationError serializer.save() # create klassset project_manager = ProjectManager() project_id = project_manager.get_project_id_by_name(name) klassset_manager = KlasssetManager() klassset = request.data.get('klasses') klassset_manager.set_klassset(project_id, user_id, klassset) # create storage storage = StorageSerializer(data={ 'storage_type': request.data.get('storage_type', None), 'storage_config': json.dumps(request.data.get('storage_config', None)), 'project': project_id }) if not storage.is_valid(): raise ValidationError storage.save() return HttpResponse(status=201, content=json.dumps({}), content_type='application/json')
def extract(cls, user_id, project_id, original_id, candidates): original = OriginalManager().get_original(project_id, original_id, status='analyzed') storage = StorageSerializer().get_storage(project_id, original['storage_id']) storage_config = copy.deepcopy(storage['storage_config']) original_path = StorageSerializer.get_original_path( storage['storage_type'], storage['storage_config'], original['name']) storage_config.update({'path': original_path}) output_dir = StorageSerializer.get_dataset_output_dir( storage['storage_type'], storage['storage_config'], original['name'], candidates) storage_config.update({'output_dir': output_dir}) automan_config = cls.__get_automan_config(user_id) automan_config.update({'path': '/projects/' + project_id + '/datasets/'}) raw_data_config = cls.__get_raw_data_config(project_id, original_id, candidates) job_config = { 'storage_type': storage['storage_type'], 'storage_config': storage_config, 'automan_config': automan_config, 'raw_data_config': raw_data_config } job_config_json = json.dumps(job_config) new_job = Job( job_type='extractor', user_id=user_id, project_id=project_id, job_config=job_config_json) new_job.save() if original['file_type'] == 'rosbag': job = RosbagExtractor(**job_config) job.create(cls.__generate_job_name(new_job.id, 'extractor')) res = job.run() return res else: raise ValidationError()
def save_file(self, project_id, original_id, file): # if target files is existed and the limit time has passed, delete it. original = self.get_original(project_id, original_id, status='registered') storage = StorageSerializer().get_storage(project_id, original['storage_id']) if storage['storage_type'] != 'LOCAL_NFS': raise ValidationError() dir_path = (storage['storage_config']['mount_path'] + storage['storage_config']['base_dir'] + '/' + original['name'] + '/raw/') # FIXME: Rule Aggregation file_path = dir_path + file.name try: os.makedirs(dir_path) except Exception: original = Original.objects.filter(id=original_id).first() if original is None: raise ObjectDoesNotExist() file_name = file.name + '_' + datetime.now().strftime('%s') dir_path = storage['storage_config']['mount_path'] +storage['storage_config']['base_dir'] + '/' + file_name + '/raw/' # FIXME: Rule Aggregation os.makedirs(dir_path) original.name = file_name original.save() file_path = dir_path + file_name # write with open(file_path, 'ab') as destination: for chunk in file.chunks(): destination.write(chunk) return {'files': [file.name]}
def archive(cls, user_id, project_id, dataset_id, original_id, annotation_id): original = OriginalManager().get_original(project_id, original_id, status='analyzed') storage = StorageSerializer().get_storage(project_id, original['storage_id']) storage_config = copy.deepcopy(storage['storage_config']) automan_config = cls.__get_automan_config(user_id) automan_config.update({'path': '/projects/' + str(project_id) + '/annotations/' + str(annotation_id) + '/'}) archive_config = cls.__get_archive_info(user_id, project_id, dataset_id, annotation_id, original_id) job_config = { 'storage_type': storage['storage_type'], 'storage_config': storage_config, 'automan_config': automan_config, 'archive_config': archive_config, } job_config_json = json.dumps(job_config) new_job = Job( job_type='archiver', user_id=user_id, project_id=project_id, job_config=job_config_json) new_job.save() job = AnnotationArchiver(**job_config) job.create(cls.__generate_job_name(new_job.id, 'archiver')) res = job.run() return res
def get_frame(request, project_id, dataset_id, candidate_id, frame): username = request.user user_id = AccountManager.get_id_by_username(username) if not Permission.hasPermission(user_id, 'get_annotationwork', project_id): raise PermissionDenied dataset = DatasetManager().get_dataset(user_id, dataset_id) original = OriginalManager().get_original(project_id, dataset['original_id']) storage = StorageSerializer().get_storage(project_id, original['storage_id']) if storage['storage_type'] == 'LOCAL_NFS': image_link = request.build_absolute_uri(request.path) + 'image/' elif storage['storage_type'] == 'AWS_S3': ext = __get_extension(candidate_id) key = (dataset['file_path'] + candidate_id + '_' + str(frame).zfill(6) + ext) image_link = AwsS3Client().get_s3_down_url( storage['storage_config']['bucket'], key) else: raise UnknownStorageTypeError frame = DatasetFrameManager.get_dataset_frame(project_id, dataset_id, frame, storage['storage_type']) content = {'image_link': image_link, 'frame': frame['frame']} return HttpResponse(status=200, content=json.dumps(content), content_type='application/json')
def annotation(request, project_id, annotation_id): username = request.user user_id = AccountManager.get_id_by_username(username) annotation_manager = AnnotationManager() if request.method == 'GET': if not Permission.hasPermission(user_id, 'get_annotationwork', project_id): raise PermissionDenied contents = annotation_manager.get_annotation(annotation_id) return HttpResponse(content=json.dumps(contents), status=200, content_type='application/json') elif request.method == 'POST': file_path = request.data.get('file_path') file_name = request.data.get('file_name') annotation_manager.set_archive(annotation_id, file_path, file_name) return HttpResponse(status=201, content=json.dumps({}), content_type='application/json') else: if not Permission.hasPermission(user_id, 'delete_annotationwork', project_id): raise PermissionDenied dataset_id = annotation_manager.get_annotation( annotation_id)['dataset_id'] original_id = DatasetManager().get_dataset(user_id, dataset_id)['original_id'] storage_id = OriginalManager().get_original(project_id, original_id)['storage_id'] storage = StorageSerializer().get_storage(project_id, storage_id) annotation_manager.delete_annotation(annotation_id, storage) return HttpResponse(status=204)
def delete_rosbag(self, project_id, user_id, original_id): rosbag = Original.objects.filter(project_id=project_id, id=original_id).first() if rosbag is None: raise ObjectDoesNotExist() storage = StorageSerializer().get_storage(project_id, rosbag.storage_id) config = storage['storage_config'] dataset_manager = DatasetManager() if dataset_manager.get_datasets_count_by_original(original_id) == 0: candidates = DatasetCandidate.objects.filter(original=original_id) for candidate in candidates: candidate.delete() rosbag.delete() if storage['storage_type'] == 'LOCAL_NFS': path = (config['mount_path'] + config['base_dir'] + '/' + rosbag.name + '/') shutil.rmtree(path) elif storage['storage_type'] == 'AWS_S3': key = config['base_dir'] + '/raws/' + rosbag.name AwsS3Client().delete_s3_files(config['bucket'], key) return True
def destroy(self, request, project_id, dataset_id): username = request.user dataset_manager = DatasetManager() user_id = AccountManager.get_id_by_username(username) if not Permission.hasPermission(user_id, 'delete_dataset', project_id): raise PermissionDenied original_id = DatasetManager().get_dataset(user_id, dataset_id)['original_id'] storage_id = OriginalManager().get_original(project_id, original_id)['storage_id'] storage = StorageSerializer().get_storage(project_id, storage_id) dataset_manager.delete_dataset(user_id, dataset_id, storage) return HttpResponse(status=204)
def delete_project(self, project_id, user_id): content = Projects.objects.filter(id=project_id).first() if content is None: raise ObjectDoesNotExist() storages = StorageSerializer().get_storages(project_id) for storage in storages: config = storage['storage_config'] if storage['storage_type'] == 'LOCAL_NFS': path = (config['mount_path'] + config['base_dir']) if os.path.isdir(path): shutil.rmtree(path) elif storage['storage_type'] == 'AWS_S3': AwsS3Client().delete_s3_files( config['bucket'], config['base_dir'] + '/') content.delete()
def save_file(self, project_id, file): # if target files is existed and the limit time has passed, delete it. storage = StorageSerializer().get_storages(project_id)[0] if storage['storage_type'] != 'LOCAL_NFS': raise ValidationError() storage_manager = StorageManager(project_id, storage['id']) dir_path = storage_manager.original_dirname file_path = storage_manager.get_original_filepath(file.name) os.makedirs(dir_path, exist_ok=True) # write with open(file_path, 'ab') as destination: for chunk in file.chunks(): destination.write(chunk) return {'files': [file.name]}
def delete_rosbag(self, project_id, user_id, original_id): rosbag = Original.objects.filter(project_id=project_id, id=original_id).first() if rosbag is None: raise ObjectDoesNotExist() storage = StorageSerializer().get_storage(project_id, rosbag.storage_id) dir_path = (storage['storage_config']['mount_path'] + storage['storage_config']['base_dir'] + '/' + rosbag.name + '/') dataset_manager = DatasetManager() if dataset_manager.get_datasets_count_by_original(original_id) == 0: candidates = DatasetCandidate.objects.filter(original=original_id) for candidate in candidates: candidate.delete() rosbag.delete() shutil.rmtree(dir_path) return True
def download_archived_link(request, project_id, annotation_id): username = request.user user_id = AccountManager.get_id_by_username(username) annotation_manager = AnnotationManager() dataset_id = annotation_manager.get_annotation(annotation_id)['dataset_id'] original_id = DatasetManager().get_dataset(user_id, dataset_id)['original_id'] storage_id = OriginalManager().get_original(project_id, original_id)['storage_id'] storage = StorageSerializer().get_storage(project_id, storage_id) if storage['storage_type'] == 'LOCAL_NFS': content = request.build_absolute_uri(request.path) + 'local/' elif storage['storage_type'] == 'AWS_S3': archive_path = annotation_manager.get_archive_path(annotation_id) content = AwsS3Client().get_s3_down_url( storage['storage_config']['bucket'], archive_path) else: raise UnknownStorageTypeError return HttpResponse(status=200, content=content, content_type='text/plain')