def create_directory(dir_data, session):
working_dir = WorkingDir()
working_dir.user_id = dir_data['user'].id
working_dir.project_id = dir_data['project'].id
if dir_data.get('jobs_to_sync'):
working_dir.jobs_to_sync = dir_data.get('jobs_to_sync')
session.add(working_dir)
regular_methods.commit_with_rollback(session)
if dir_data.get('files'):
file_list = dir_data.get('files')
for file in file_list:
WorkingDirFileLink.add(session, working_dir.id, file)
regular_methods.commit_with_rollback(session)
return working_dir
def generate_sample_dataset(self, nickname, project):
existing_dir = self.session.query(WorkingDir).filter(
WorkingDir.nickname == nickname,
WorkingDir.project_id == project.id).first()
if existing_dir:
return existing_dir, True
# Reusing new blank directory.
working_dir = WorkingDir.new_blank_directory(session=self.session,
project=project,
project_id=project.id,
nickname=nickname)
self.session.add(working_dir)
self.session.flush()
Project_Directory_List.add(session=self.session,
working_dir_id=working_dir.id,
project_id=project.id,
nickname=nickname)
project.set_cache_key_dirty('directory_list')
self.session.add(project)
return working_dir, False
def new(
session,
name: str,
project_string_id: str,
goal: str,
user,
member_created,
):
# Avoid circular import
# Maybe in future Project_permissions class could be part of project
from shared.permissions.project_permissions import Project_permissions
project = Project( # Base object
name=name,
project_string_id=project_string_id,
goal=goal,
user_primary=user, # Assumed to be primary
member_created=member_created)
# Permissions and user associations
user.projects.append(project)
user.current_project_string_id = project_string_id
user.project_current = project
permission_add_result, permission_add_error_message = Project_permissions.add(
permission="admin", user=user, sub_type=project_string_id)
session.add(user, project)
session.flush()
member_id = user.member_id
Event.new(
session=session,
kind="new_project",
member_id=member_id,
success=True,
project_id=project.id,
email=user.email # Caution, assumes user object is available
)
project.directory_default = WorkingDir.new_user_working_dir(
session, None, project, user, project_default_dir=True)
report_dashboard = ReportDashboard.new(project_id=project.id)
session.add(report_dashboard)
session.flush() # Needed to work with adding default directory
# careful this expects a default dir already assigned
Project_Directory_List.add_default(
session=session,
working_dir_id=project.directory_default.id,
project=project)
return project
def register_user(user_data, session):
"""
:param user_data:
Example
user_data={
'username': '******',
'email': '*****@*****.**',
'password': '******',
'project_string_id': 'myproject'
}
:return:
"""
# User registration
password_hash = hashing_functions.make_password_hash(
user_data['email'],
user_data['password'])
# TODO could insert logic here to attach a user to a project based on say the sigup code
# ie signup_code.something?
username = user_data['username']
user_email = user_data['email']
new_user = User(
email = user_email,
password_hash = password_hash,
username = username,
api_enabled_builder = True,
api_enabled_trainer = True,
security_email_verified = True,
last_builder_or_trainer_mode = 'builder',
permissions_general = {'general': ['normal_user']}
)
new_user.permissions_projects = {} # I don't like having this here but alternative of committing object seems worse
session.add(new_user)
if 'project_string_id' in user_data:
new_user.current_project_string_id = user_data['project_string_id']
project = Project.get(session, user_data['project_string_id'])
if project is not None:
new_user.projects.append(project)
new_user_working_dir = WorkingDir.new_user_working_dir(
session,
None,
project,
new_user
)
permission_result, permission_error = Project_permissions.add(user_data['project_string_id'],
new_user,
user_data['project_string_id'])
return new_user
def generate_data():
spec_list = [{
'data_type': {
'default': 'dataset',
'kind': str,
'required': False
},
}, {
'dataset_id': {
'default': None,
'kind': int,
'required': False
},
}, {
'project_id': {
'default': None,
'kind': int,
'required': False
}
}, {
'structure': {
'default': None,
'kind': str,
'required': False
}
}]
log, input, untrusted_input = regular_input.master(request=request,
spec_list=spec_list)
with sessionMaker.session_scope() as session:
data_mocker = DiffgramDataMocker(session=session)
if input['data_type'] == 'dataset':
dataset = WorkingDir.get(session=session,
directory_id=input['dataset_id'],
project_id=input['project_id'])
data_mocker.generate_test_data_on_dataset(dataset=dataset)
elif input['data_type'] == 'label':
project = Project.get_by_id(session, id=input['project_id'])
data_mocker.generate_sample_label_files(project=project)
elif input['data_type'] == 'task_template':
project = Project.get_by_id(session, id=input['project_id'])
data_mocker.generate_test_data_for_task_templates(
project=project, structure=input.get('structure'))
elif input['data_type'] == 'project':
user = User.get(session=session)
data_mocker.generate_sample_project(user=user)
return jsonify({'message': 'OK.'})
def web_build_name_to_file_id_dict(project_string_id):
"""
Given we know a label_name, and where we are working,
return the label_file_id
Arguments:
project_string_id, integer
working_dir_id, integer
Returns:
dict of label files
or None / failure case
"""
log = {"success": False, "errors": []}
directory_id = request.headers.get('directory_id', None)
if directory_id is None:
log["errors"].append("'directory_id' not supplied")
return jsonify(log), 200
with sessionMaker.session_scope() as session:
project = Project.get(session, project_string_id)
verify_result = WorkingDir.verify_directory_in_project(
session, project, directory_id)
if verify_result is False:
log["errors"].append("Invalid directory id")
log["success"] = False
return jsonify(log=log), 200
name_to_file_id, result = build_name_to_file_id_dict(
session=session, directory_id=directory_id)
if result == True:
log["success"] = True
return jsonify(log=log, name_to_file_id=name_to_file_id), 200
def __create_sample_task_template(self, name, project):
task_template = Job()
task_template.name = name
task_template.permission = 'all_secure_users'
task_template.project_id = project.id
task_template.instance_type = 'box'
task_template.stat_count_tasks = 0
task_template.label_mode = 'closed_all_available'
task_template.stat_count_complete = 0
directory = WorkingDir.new_blank_directory(session=self.session)
task_template.directory = directory
label_files = self.generate_sample_label_files(project=project)
task_template.label_dict = {
'label_file_list': [
x.serialize_with_label_and_colour(self.session)['id']
for x in label_files
]
}
task_template.status = 'active'
self.session.add(task_template)
self.session.flush()
task_template_label_attach(self.session, task_template)
return task_template
def create_job(job_data, session):
"""
The function will create a Job object for testing purposes. You can supply you own
project if there is one in specific that has to be attached, if not the function will
also mock a project for you. The idea is that new developer don't have to worry about the
relations between object in order to mock faster.
For example: "I want to create a Job, but I don't really care about the project."
- Then I should not worry about the details of mocking a project.
TODO:
More data mocks are still pending, like members, labels, etc...
For now, this is sufficient for the current tests we're writing.
:param job_data:
:param session:
:return:
"""
if job_data.get('project'):
job = Job(member_created = None, project = job_data.get('project'))
else:
project_string_id = '{}-job-project'.format(get_random_string(8))
project_context = {
'project_string_id': project_string_id,
'project_name': '{}-job-project'.format(project_string_id),
'users': [
{'username': '******'.format(get_random_string(5)),
'email': '*****@*****.**',
'password': '******',
'project_string_id': project_string_id
}
]
}
project_data = create_project_with_context(project_context, session)
job = Job(member_created = None, project = project_data.get('project'),
project_id = project_data.get('project').id)
session.add(job)
# TODO: support mocking labels.
job.label_dict = job_data.get('label_dict', {})
job.type = job_data.get('type', 'draft')
job.status = job_data.get('status', 'draft')
job.label_dict['label_file_list'] = job_data.get('label_file_list', [])
job.name = job_data.get('name', None)
job.output_dir_action = job_data.get('output_dir_action', 'nothing')
job.share_type = job_data.get('name', 'project')
job.share_type = job.share_type.lower()
job.launch_datetime = datetime.datetime.now()
if job.launch_datetime is not None:
job.waiting_to_be_launched = True
job.interface_connection_id = job_data.get('interface_connection_id')
job.file_count = job_data.get('file_count', 0) # note this is user set
job.permission = job_data.get('file_count', 'all_secure_users')
job.label_mode = job_data.get('label_mode', 'open')
job.passes_per_file = job_data.get('passes_per_file', 1)
job.instance_type = job_data.get('instance_type', 'box')
job.file_handling = job_data.get('file_handling', 'use_existing')
job.stat_count_tasks = job_data.get('stat_count_tasks', 0)
job.completion_directory_id = job_data.get('completion_directory_id', None)
directory = WorkingDir.new_blank_directory(session = session)
session.add(directory)
job.directory = directory
# if job.share_type == "market":
# bid_new_core(
# session=session,
# job=job,
# )
regular_methods.commit_with_rollback(session)
for dir in job_data.get('attached_directories', []):
rel = JobWorkingDir()
rel.sync_type = 'sync'
rel.job_id = job.id
rel.working_dir_id = dir.id
session.add(rel)
regular_methods.commit_with_rollback(session)
return job