def __fetch_object(self, opts):
bucket = self.connection_client.get_bucket(opts['bucket_name'])
blob = bucket.blob(opts['path'])
blob_expiry = int(time.time() + (60 * 60 * 24 * 30))
signed_url = blob.generate_signed_url(expiration=blob_expiry)
# Deduct Media Type:
# TODO Share this with existing process_media determine_media_type()
extension = Path(opts['path']).suffix
extension = extension.lower()
media_type = None
if extension in images_allowed_file_names:
media_type = 'image'
elif extension in videos_allowed_file_names:
media_type = 'video'
else:
# TODO: Decide, do we want to raise an exception? or just do nothing?
log = regular_log.default()
log['error']['invalid_type'] = 'File must type of: {} {}'.format(
str(images_allowed_file_names), str(videos_allowed_file_names))
log['error']['file_name'] = opts['path']
log['opts'] = opts
with sessionMaker.session_scope() as session:
Event.new(session=session,
member_id=opts['event_data']['request_user'],
kind='google_cloud_new_import_error',
description='New cloud import for {}'.format(
opts['path']),
error_log=log)
raise LookupError('File must type of: {} {}'.format(
str(images_allowed_file_names),
str(videos_allowed_file_names)))
# metadata = self.connection_client.head_object(Bucket=opts['bucket_name, Key=path)
with sessionMaker.session_scope() as session:
created_input = packet.enqueue_packet(
self.config_data['project_string_id'],
session=session,
media_url=signed_url,
media_type=media_type,
job_id=opts.get('job_id'),
video_split_duration=opts.get('video_split_duration'),
directory_id=opts.get('directory_id'))
log = regular_log.default()
log['opts'] = opts
Event.new(session=session,
member_id=opts['event_data']['request_user'],
kind='google_cloud_new_import_success',
description='New cloud import for {}'.format(
opts['path']),
error_log=opts)
return {'result': created_input}
def test():
project_string_id = "sdk-test"
permission_level = "Editor"
with sessionMaker.session_scope() as session:
auth = new.create(session, project_string_id, permission_level)
print(auth.serialize_with_secret())
def test(): project_string_id = "" client_id = "" #client_id = None #client_id = "123" with sessionMaker.session_scope() as session: result = revoke.by_client_id(session, project_string_id, client_id) print(result)
def user_view():
with sessionMaker.session_scope() as s:
user = User.get(s)
if user is None:
return jsonify({"none_found": True}), 400, {
'ContentType': 'application/json'
}
out = jsonify(user=user.serialize())
return out, 200, {'ContentType': 'application/json'}
def verify_otp_from_web():
with sessionMaker.session_scope() as session:
data = request.get_json(force=True)
proposed_otp_code = data.get('otp', None)
otp_current_session = data.get('otp_current_session', None)
email = data.get('email', None)
user = User.get_by_email(session, email)
if user is None:
return jsonify(error="No user"), 200, {
'ContentType': 'application/json'
}
if user.otp_current_session_expiry <= time.time():
User.new_login_history(session=session,
success=False,
otp_success=False,
remote_address=request.remote_addr,
user_id=user.id)
return jsonify(error="Please login again, session expired"), 200, {
'ContentType': 'application/json'
}
if user.otp_current_session != otp_current_session:
User.new_login_history(session=session,
success=False,
otp_success=False,
remote_address=request.remote_addr,
user_id=user.id)
return jsonify(error="Please login again, session invalid"), 200, {
'ContentType': 'application/json'
}
if OneTimePass.verify_otp(user, proposed_otp_code) is True:
User.new_login_history(session=session,
success=True,
otp_success=True,
remote_address=request.remote_addr,
user_id=user.id)
setSecureCookie(user)
return jsonify(user=user.serialize(), success=True), 200, {
'ContentType': 'application/json'
}
return jsonify(error="Invalid code"), 200, {
'ContentType': 'application/json'
}
def generate_mask_by_project_id(project_string_id):
# TODO use a thread, this is a long running process
semantic_segmentation_data_prep = Semantic_segmentation_data_prep()
with sessionMaker.session_scope() as session:
project = Project.get(session, project_string_id)
type = "joint"
# type = "binary"
semantic_segmentation_data_prep.generate_mask_core(
session, project, type)
return "ok", 200, {'ContentType': 'application/json'}
def tag_view_all_public():
with sessionMaker.session_scope() as session:
tag_list = session.query(Tag).filter(Tag.is_public == True).all()
tag_list_serailized = []
for tag in tag_list:
tag_list_serailized.append(tag.name)
out = jsonify(success=True, tag_list=tag_list_serailized)
return out, 200, {'ContentType': 'application/json'}
def tag_view_by_project(project_string_id):
with sessionMaker.session_scope() as session:
project = Project.get_project(session, project_string_id)
if project is None:
return jsonify(success=False), 400, {
'ContentType': 'application/json'
}
tag_list = project.serialize_tag_list_PUBLIC()
out = jsonify(success=True, tag_list=tag_list)
return out, 200, {'ContentType': 'application/json'}
def disable_otp_from_web():
with sessionMaker.session_scope() as session:
user = User.get(session)
if not user:
return "no user", 400, {'ContentType': 'application/json'}
session.add(user)
user.otp_enabled = False
out = jsonify(success=True, user=user.serialize())
return out, 200, {'ContentType': 'application/json'}
def user_exists(email):
if settings.DIFFGRAM_SYSTEM_MODE not in [
'testing_e2e', 'testing', 'sandbox'
]:
return jsonify(message='Invalid System Mode'), 400
with sessionMaker.session_scope() as session:
user = User.get_by_email(session, email)
if user is None:
return jsonify({"none_found": True}), 400, {
'ContentType': 'application/json'
}
out = jsonify(found=True)
return out, 200, {'ContentType': 'application/json'}
def by_project_core(project_string_id: str,
Roles: list,
apis_project_list: list = [],
apis_user_list: list = []):
with sessionMaker.session_scope() as session:
if not project_string_id or project_string_id == "null" or project_string_id == "undefined":
raise Forbidden(default_denied_message)
if request.authorization is not None:
result = API_Permissions.by_project(
session=session,
project_string_id=project_string_id,
Roles=Roles)
if result is not True:
raise Forbidden(default_denied_message +
" API Permissions")
# At the moment auth doesn't actually
# get project as it has all results stored...
# not clear that we would need a None check here
# given its checked in auth?
project = Project.get(session, project_string_id)
if project is None:
raise Forbidden(default_denied_message +
" Can't find project")
# Project APIs, maybe should role this into API_Permissions
check_all_apis(project=project,
apis_required_list=apis_project_list)
return True
result = Project_permissions.check_permissions(
session=session,
apis_project_list=apis_project_list,
apis_user_list=apis_user_list,
project_string_id=project_string_id,
Roles=Roles)
if result is True:
return True
else:
raise Forbidden(default_denied_message)
def enable_otp_from_web():
with sessionMaker.session_scope() as session:
user = User.get(session)
if not user:
return "no user", 400, {'ContentType': 'application/json'}
otp, qr_code_url, backup_code_list = OneTimePass.new(session, user)
out = jsonify(success=True,
otp=otp,
qr_code_url=qr_code_url,
backup_code_list=backup_code_list,
user=user.serialize())
return out, 200, {'ContentType': 'application/json'}
def system_startup_events_check(service_name):
"""
Checks multiple system settings changes and logs them if any changes detected.
:param session:
:return:
"""
if settings.DIFFGRAM_INSTALL_FINGERPRINT is None:
logger.error(
'DIFFGRAM_INSTALL_FINGERPRINT is not Set as an ENV variable. \n'
'Please set it by running install.py script again. Value will be stored in .env file.')
return False
if settings.DIFFGRAM_VERSION_TAG is None:
logger.error(
'DIFFGRAM_VERSION_TAG is not Set as an ENV variable. \n'
'Please set it by running install.py script again. Value will be stored in .env file.')
return False
if settings.DIFFGRAM_HOST_OS is None:
logger.error(
'DIFFGRAM_HOST_OS is not Set as an ENV variable. \n'
'Please set it by running install.py script again. Value will be stored in .env file.')
return False
with session_scope() as session:
# Record Startup Time
SystemEvents.new(
session = session,
kind = 'system_startup',
description = 'Diffgram System startup for {} service'.format(service_name),
install_fingerprint = settings.DIFFGRAM_INSTALL_FINGERPRINT,
previous_version = None,
diffgram_version = settings.DIFFGRAM_VERSION_TAG,
host_os = settings.DIFFGRAM_HOST_OS,
storage_backend = settings.DIFFGRAM_STATIC_STORAGE_PROVIDER,
service_name = service_name,
startup_time = datetime.datetime.utcnow(),
shut_down_time = None,
created_date = datetime.datetime.utcnow()
)
SystemEvents.check_version_upgrade(session = session, service_name = service_name)
SystemEvents.check_os_change(session = session, service_name = service_name)
return True
def annotation_example_toggle(project_string_id):
with sessionMaker.session_scope() as s:
data = request.get_json(force=True)
reqiest_image = data.get('image', None)
if reqiest_image is None:
return json.dumps("image is None"), 400, {
'ContentType': 'application/json'
}
request_image_id = reqiest_image.get('id', None)
if request_image_id is None:
return json.dumps("image_id is None"), 400, {
'ContentType': 'application/json'
}
image = Image.get_by_id(s, request_image_id)
image.is_annotation_example = not image.is_annotation_example
s.add(image)
out = {'success': True}
return jsonify(out), 200, {'ContentType': 'application/json'}
def web_build_name_to_file_id_dict(project_string_id):
"""
Given we know a label_name, and where we are working,
return the label_file_id
Arguments:
project_string_id, integer
working_dir_id, integer
Returns:
dict of label files
or None / failure case
"""
log = {"success": False, "errors": []}
directory_id = request.headers.get('directory_id', None)
if directory_id is None:
log["errors"].append("'directory_id' not supplied")
return jsonify(log), 200
with sessionMaker.session_scope() as session:
project = Project.get(session, project_string_id)
verify_result = WorkingDir.verify_directory_in_project(
session, project, directory_id)
if verify_result is False:
log["errors"].append("Invalid directory id")
log["success"] = False
return jsonify(log=log), 200
name_to_file_id, result = build_name_to_file_id_dict(
session=session, directory_id=directory_id)
if result == True:
log["success"] = True
return jsonify(log=log, name_to_file_id=name_to_file_id), 200
def __fetch_object(self, opts):
"""Upload a file to diffgram from an S3 bucket
:param s3_file_key: path of file to fetch from
:return: file obj if file was uploaded, else False
"""
spec_list = [{'bucket_name': str, 'path': str}]
log = regular_log.default()
log, input = regular_input.input_check_many(untrusted_input=opts,
spec_list=spec_list,
log=log)
if len(log["error"].keys()) >= 1:
return {'log': log}
# This might be an issue. Currently not supporting urls with no expiration. Biggest time is 1 week.
signed_url = self.connection_client.generate_presigned_url('get_object',
Params={'Bucket': opts['bucket_name'],
'Key': opts['path']},
ExpiresIn=3600 * 24 * 6) # 5 Days.
with sessionMaker.session_scope() as session:
project = Project.get_by_string_id(session, self.config_data.get('project_string_id'))
member = session.query(Member).filter(Member.user_id == opts['event_data']['request_user']).first()
# Deduct Media Type:
extension = Path(opts['path']).suffix
extension = extension.lower()
media_type = None
if extension in images_allowed_file_names:
media_type = 'image'
elif extension in videos_allowed_file_names:
media_type = 'video'
else:
# TODO: Decide, do we want to raise an exception? or just do nothing?
log = regular_log.default()
log['error']['invalid_type'] = 'File must type of: {} {}'.format(str(images_allowed_file_names),
str(videos_allowed_file_names))
log['error']['file_name'] = opts['path']
log['opts'] = opts
Event.new(
session=session,
member_id=opts['event_data']['request_user'],
kind='aws_s3_new_import_warning',
description='Skipped import for {}, invalid file type.'.format(opts['path']),
error_log=log,
project_id=project.id,
member=member,
success=False
)
return None
# print('AAAAA', opts, opts.get('job_id'))
# metadata = self.connection_client.head_object(Bucket=self.config_data['bucket_name'], Key=path)
created_input = packet.enqueue_packet(self.config_data['project_string_id'],
session=session,
media_url=signed_url,
media_type=media_type,
job_id=opts.get('job_id'),
video_split_duration=opts.get('video_split_duration'),
directory_id=opts.get('directory_id'))
log = regular_log.default()
log['opts'] = opts
Event.new(
session=session,
member_id=opts['event_data']['request_user'],
kind='aws_s3_new_import_success',
description='New cloud import for {}'.format(opts['path']),
error_log=opts,
project_id=project.id,
member=member,
success=True
)
return created_input
def inner(*args, **kwds):
job_id = kwds.get('job_id', None)
if job_id is None or job_id == "null" or job_id == "undefined":
raise Forbidden("job_id is invalid")
with sessionMaker.session_scope() as session:
# Permissions cascading from project
project_string_id = get_project_string_from_job_id(
session, job_id)
# API
if request.authorization is not None:
result = API_Permissions.by_project(
session=session,
project_string_id=project_string_id,
Roles=project_role_list)
if result is True:
return func(*args, **kwds)
else:
raise Forbidden("API access invalid")
# TODO do we need to validate user has applicable mode?
# ie they pass mode builder but are trainer?
# Basics should fail on project level check anyway here...
# User
# TODO move login stuff into the general User_Permissions
if LoggedIn() != True:
raise Forbidden("Login again.")
user = session.query(User).filter(
User.id == getUserID()).first()
if user is None:
raise Forbidden("Login again.")
# Want to use the builder API permissions instead of
# flags since a user may be testing this as a builder
# TODO deprecate 'mode' flag or have it as something else
# like "builder_only" or something
# Jan 3, 2020
# One downside of doing it this way is it means
# that we need to be careful with
# project_role_list list...
if user.api_enabled_builder is True:
result = Project_permissions.check_permissions(
session=session,
project_string_id=project_string_id,
Roles=project_role_list,
apis_project_list=apis_project_list,
apis_user_list=apis_user_list)
if result is True:
return func(*args, **kwds)
else:
raise Forbidden("Project access invalid")
if user.api_enabled_trainer is True:
# TODO refactor into function
# TODO handle "info" case of a trainer not yet
# on a job seeing basic stuff on active jobs...
# We allow trainers to see
# Basic info before they apply
# as long as job is active...
#if job.status != "active":
# raise Forbidden("No access.")
User_Permissions.general(user=user,
apis_user_list=apis_user_list)
user_to_job = User_To_Job.get_single_by_ids(
session=session, user_id=user.id, job_id=job_id)
# TODO other status checking on this...
if user_to_job is None:
raise Forbidden(
"No access to this job. Please apply first.")
# Success case for trainer
return func(*args, **kwds)
raise Forbidden("No access.")
def __fetch_object(self, opts):
"""
Upload a file to Diffgram from an Azure Blob
:param opts: Dictionary with parameters for object fetching.
:return: file obj if file was uploaded, else False
"""
spec_list = [{'bucket_name': str, 'path': str}]
log = regular_log.default()
log, input = regular_input.input_check_many(untrusted_input = opts,
spec_list = spec_list,
log = log)
if len(log["error"].keys()) >= 1:
return {'log': log}
shared_access_signature = BlobSharedAccessSignature(
account_name = self.connection_client.account_name,
account_key = self.connection_client.credential.account_key
)
expiration_offset = 40368000
blob_name = opts['path']
container = opts['bucket_name']
added_seconds = datetime.timedelta(0, expiration_offset)
expiry_time = datetime.datetime.utcnow() + added_seconds
filename = blob_name.split("/")[-1]
sas = shared_access_signature.generate_blob(
container_name = container,
blob_name = blob_name,
start = datetime.datetime.utcnow(),
expiry = expiry_time,
permission = BlobSasPermissions(read = True),
content_disposition = 'attachment; filename=' + filename,
)
sas_url = 'https://{}.blob.core.windows.net/{}/{}?{}'.format(
self.connection_client.account_name,
container,
blob_name,
sas
)
with sessionMaker.session_scope() as session:
project = Project.get_by_string_id(session, self.config_data.get('project_string_id'))
member = session.query(Member).filter(Member.user_id == opts['event_data']['request_user']).first()
# Deduct Media Type:
extension = Path(opts['path']).suffix
extension = extension.lower()
media_type = None
if extension in images_allowed_file_names:
media_type = 'image'
elif extension in videos_allowed_file_names:
media_type = 'video'
else:
# TODO: Decide, do we want to raise an exception? or just do nothing?
log = regular_log.default()
log['error']['invalid_type'] = 'File must type of: {} {}'.format(str(images_allowed_file_names),
str(videos_allowed_file_names))
log['error']['file_name'] = opts['path']
log['opts'] = opts
Event.new(
session = session,
member_id = opts['event_data']['request_user'],
kind = 'microsoft_azure_new_import_warning',
description = 'Skipped import for {}, invalid file type.'.format(opts['path']),
error_log = log,
project_id = project.id,
member = member,
success = False
)
return None
# metadata = self.connection_client.head_object(Bucket=self.config_data['bucket_name'], Key=path)
created_input = packet.enqueue_packet(self.config_data['project_string_id'],
session = session,
media_url = sas_url,
media_type = media_type,
job_id = opts.get('job_id'),
batch_id = opts.get('batch_id'),
file_name = opts.get('path'),
video_split_duration = opts.get('video_split_duration'),
directory_id = opts.get('directory_id'),
extract_labels_from_batch = True)
log = regular_log.default()
log['opts'] = opts
Event.new(
session = session,
member_id = opts['event_data']['request_user'],
kind = 'microsoft_azure_new_import_success',
description = 'New cloud import for {}'.format(opts['path']),
error_log = opts,
project_id = project.id,
member = member,
success = True
)
return created_input
def __send_export(self, opts):
spec_list = [{'project_string_id': dict}]
log = regular_log.default()
log, input = regular_input.input_check_many(untrusted_input = self.config_data,
spec_list = spec_list,
log = log)
if len(log["error"].keys()) >= 1:
return {'log': log}
spec_list = [
{'path': str},
{"format": {
'default': 'JSON',
'kind': str,
'valid_values_list': ['JSON', 'YAML']
}},
{'export_id': str},
{'bucket_name': str},
]
log = regular_log.default()
log, input = regular_input.input_check_many(untrusted_input = opts,
spec_list = spec_list,
log = log,
string_len_not_zero = False)
if len(log["error"].keys()) >= 1:
return {'log': log}
if not opts['path'].endswith('/') and opts['path'] != '':
log['error']['path'] = 'Path on bucket must be a folder, not a filename.'
return log
with sessionMaker.session_scope() as session:
project = Project.get_by_string_id(session, self.config_data['project_string_id'])
member = session.query(Member).filter(Member.user_id == opts['event_data']['request_user']).first()
export = session.query(Export).filter(Export.id == opts['export_id']).first()
# Check perms and export status.
export_check_result = check_export_permissions_and_status(export,
self.config_data['project_string_id'],
session)
if len(export_check_result['error'].keys()) > 1:
return export_check_result
result = export_view_core(
export = export,
format = opts['format'],
return_type = 'bytes')
filename = generate_file_name_from_export(export, session)
if opts['path'] != '':
key = '{}{}.{}'.format(opts['path'], filename, opts['format'].lower())
else:
key = '{}.{}'.format(filename, opts['format'].lower())
file = io.BytesIO(result)
blob_client = self.connection_client.get_blob_client(container = opts['bucket_name'], blob = key)
content_type = mimetypes.guess_type(filename)[0]
my_content_settings = ContentSettings(content_type = content_type)
blob_client.upload_blob(file, content_settings = my_content_settings)
log = regular_log.default()
log['opts'] = opts
Event.new(
session = session,
member_id = opts['event_data']['request_user'],
kind = 'microsoft_azure_new_export_success',
description = 'New cloud export for {}{}'.format(opts['path'], filename),
error_log = opts,
member = member,
project_id = project.id,
success = True
)
return {'result': True}
def __fetch_folder(self, opts):
result = []
if self.config_data.get('project_string_id') is None:
return {'result': 'error'}
paths = opts['path']
if type(paths) != list:
paths = [paths]
with sessionMaker.session_scope() as session:
project = Project.get_by_string_id(
session, self.config_data.get('project_string_id'))
member = session.query(Member).filter(
Member.user_id == opts['event_data']['request_user']).first()
for path in paths:
blobs = self.connection_client.list_blobs(opts['bucket_name'],
prefix=path)
for blob in blobs:
# Deduct Media Type:
if blob.name.endswith('/'):
continue
blob_expiry = int(time.time() + (60 * 60 * 24 * 30))
signed_url = blob.generate_signed_url(
expiration=blob_expiry)
extension = Path(blob.path).suffix
media_type = None
if extension in images_allowed_file_names:
media_type = 'image'
elif extension in videos_allowed_file_names:
media_type = 'video'
else:
logging.warn('File: {} must type of: {} {}'.format(
blob.name, str(images_allowed_file_names),
str(videos_allowed_file_names)))
log = regular_log.default()
log['error'][
'invalid_type'] = 'File must type of: {} {}'.format(
str(images_allowed_file_names),
str(videos_allowed_file_names))
log['error']['file_name'] = path
log['opts'] = opts
Event.new(
session=session,
member_id=opts['event_data']['request_user'],
kind='google_cloud_new_import_warning',
description=
'Skipped import for {}, invalid file type.'.format(
blob.name),
error_log=log,
project_id=project.id,
member=member,
success=False)
continue
result = []
# TODO: check Input() table for duplicate file?
created_input = packet.enqueue_packet(
self.config_data['project_string_id'],
session=session,
media_url=signed_url,
media_type=media_type,
job_id=opts.get('job_id'),
batch_id=opts.get('batch_id'),
file_name=path,
video_split_duration=opts.get('video_split_duration'),
directory_id=opts.get('directory_id'),
extract_labels_from_batch=True)
log = regular_log.default()
log['opts'] = opts
Event.new(session=session,
member_id=opts['event_data']['request_user'],
kind='google_cloud_new_import_success',
description='New cloud import for {}'.format(
blob.name),
error_log=opts,
project_id=project.id,
member=member,
success=True)
result.append(created_input)
return result
def update_tags(project_string_id):
"""
Update tags to latest
"""
have_error = False
error_message_list = []
with sessionMaker.session_scope() as session:
data = request.get_json(force=True)
tag_list = data.get('tag_list', None)
# tag_list could be none, ie deleted all tags...
#if tag_list is None:
#error_message_list.append("tag list is None")
#return jsonify(error_message_list), 400, {'ContentType' : 'application/json'}
project = Project.get(session, project_string_id)
print(tag_list)
rebuilt_tag_list = []
for tag in tag_list:
if valid_tag(tag):
name = tag.lower()
#Check if tag with same name already exists
#If so can just add that database object to
tag_db = session.query(Tag).filter(Tag.name == name).first()
if not tag_db:
tag_db = Tag()
tag_db.name = name
tag_db.is_public = project.is_public
session.add(tag_db)
if tag_db:
session.add(tag_db)
# TODO handle counts properly ie on tag being removed etc.
#tag_db.count += 1
rebuilt_tag_list.append(tag_db)
session.add(project)
# This handles removing link to tag that's no longer in project
project.tag_list = rebuilt_tag_list
return json.dumps({'success': True}), 200, {
'ContentType': 'application/json'
}
