def get_source_imports(source_id=None):
user = get_user()
if not permitted(user, 'get', 'source', source_id):
return "Not authorized", 401
source = db_session.query(Source).get(source_id) if source_id else None
if source == None:
return "Not found", 404
db_session.execute("SET time_zone = '+00:00'")
rows = db_session.execute("""SELECT
data_import.id,
data_import.filename,
data_import.data_type,
data_import_status.code AS status,
data_import.time_created,
data_import.upload_uuid
FROM data_import
LEFT JOIN data_import_status ON data_import_status.id = data_import.status_id
WHERE data_import.source_id = :source_id
""", { 'source_id': source_id })
return jsonify_rows(rows)
def create_or_update_program(program_id=None):
action = 'update' if program_id else 'create'
user = get_user()
if not permitted(user, action, 'program', program_id):
return "Not authorized", 401
if program_id:
program = db_session.query(MonitoringProgram).get(program_id)
else:
program = MonitoringProgram()
if not program:
return "Not found", 400
body = request.json
errors = validate_fields(program_fields, body)
if len(errors):
return jsonify(errors), 400
update_program_from_json(program, body)
db_session.add(program)
db_session.flush()
if action == 'create':
db_session.execute("""INSERT INTO user_program_manager (user_id, monitoring_program_id) VALUES (:user_id, :program_id)""",
{ 'program_id': program.id, 'user_id': user.id })
db_session.commit()
return jsonify(program_to_json(program)), 200 if program_id else 201
def get_source_processing_notes(source_id=None):
user = get_user()
if not permitted(user, 'get', 'source', source_id):
return "Not authorized", 401
source = db_session.query(Source).get(source_id) if source_id else None
if source == None:
return "Not found", 404
db_session.execute("SET time_zone = '+00:00'")
rows = db_session.execute("""SELECT
notes.id,
user.first_name,
user.last_name,
user.email,
notes.source_id,
notes.time_created,
notes.notes,
user.id = :user_id AS editable
FROM data_processing_notes notes
JOIN user ON notes.user_id = user.id
WHERE notes.source_id = :source_id
""", { 'source_id': source_id, 'user_id': user.id })
return jsonify_rows(rows)
def add_manager_to_monitoring_program(program_id = None):
if program_id == None:
return "Not found", 404
user = get_user()
if not permitted(user, 'mangage_managers', 'program', program_id):
return "Not authorized", 401
body = request.json
email = body["email"]
if not re.match(email_regex, email):
return jsonify({ 'error' : '"%s" is not a valid email address' % email }), 400
manager = db_session.query(User).filter(User.email == email).one_or_none()
if not manager:
manager = User(email=email)
db_session.add(manager)
db_session.flush()
db_session.execute("""REPLACE INTO user_program_manager (user_id, monitoring_program_id) VALUES (:user_id, :program_id)""",
{ 'user_id': manager.id, 'program_id': program_id })
db_session.commit()
return "OK", 201
def delete_source_custodian(source_id=None, user_id=None):
user = get_user()
if not permitted(user, 'manage_custodians', 'source', source_id):
return "Not authorized", 401
db_session.execute("""DELETE FROM user_source
WHERE user_id = :user_id
AND source_id = :source_id""", { 'source_id': source_id, 'user_id': user_id })
db_session.commit()
return "OK", 200
def process_import_async(import_id, status):
info = load_import(import_id)
user = get_user()
file_path = get_upload_path(info.upload_uuid)
working_path = import_path(import_id)
os.makedirs(working_path, exist_ok=True)
data_type = info.data_type
with lock:
running_imports[import_id] = {
'started': local_iso_datetime(),
'status': status,
'total_rows': 0,
'processed_rows': 0,
'is_admin': 'Administrator' in get_roles(user)
}
def result_callback(result):
is_admin = running_imports[import_id]['is_admin']
with lock:
del running_imports[import_id]
success = result['errors'] == 0
if status == 'checking':
new_status = 'checked_ok' if success else 'checked_error'
elif status == 'importing':
new_status = ('approved' if is_admin else 'imported') if success else 'import_error'
info = load_import(import_id)
info.status_id = status_ids[new_status]
info.error_count = result['errors']
info.warning_count = result['warnings']
db_session.commit()
def progress_callback(processed_rows, total_rows):
with lock:
running_imports[import_id]['total_rows'] = total_rows
running_imports[import_id]['processed_rows'] = processed_rows
try:
# Start import process
t = Thread(target = process_import, args = (file_path, working_path, data_type, status == 'importing', progress_callback, result_callback, info.source_id, import_id))
t.start()
except:
traceback.print_exc()
result_callback({
'warnings': 0,
'errors': 1
})
def delete_source(source_id=None):
user = get_user()
if not permitted(user, 'delete', 'source', source_id):
return "Not authorized", 401
db_session.execute("""DELETE FROM user_source WHERE source_id = :source_id""", { 'source_id': source_id })
db_session.execute("""DELETE FROM data_import WHERE source_id = :source_id""", { 'source_id': source_id })
db_session.execute("""DELETE FROM source WHERE id = :source_id""", { 'source_id': source_id })
remove_orphaned_monitoring_programs()
db_session.commit()
return "OK", 200
def remove_manager_from_monitoring_program(program_id = None, user_id = None):
if program_id == None or user_id == None:
return "Not found", 404
user = get_user()
if not permitted(user, 'mangage_managers', 'program', program_id):
return "Not authorized", 401
db_session.execute("""DELETE FROM user_program_manager WHERE user_id = :user_id AND monitoring_program_id = :program_id""",
{ 'program_id': program_id, 'user_id': user_id })
db_session.commit()
return "OK", 200
def remove_source_from_monitoring_program(program_id = None, source_id = None):
if program_id == None or source_id == None:
return "Not found", 404
user = get_user()
if not permitted(user, 'update', 'source', program_id):
return "Not authorized", 401
db_session.execute("""UPDATE source SET monitoring_program_id = NULL WHERE id = :source_id AND monitoring_program_id = :program_id""",
{ 'program_id': program_id, 'source_id': source_id })
db_session.commit()
return "OK", 200
def delete_monitoring_program(program_id = None):
if program_id == None:
return "Not found", 404
user = get_user()
if not permitted(user, 'delete', 'program', program_id):
return "Not authorized", 401
db_session.execute("""DELETE FROM monitoring_program WHERE id = :program_id""", { 'program_id': program_id })
db_session.commit()
return "OK", 200
def delete_source_processing_notes(source_id=None, note_id=None): user = get_user() if not permitted(user, 'delete', 'source', source_id): return "Not authorized", 401 notes = db_session.query(DataProcessingNotes).get(note_id) if notes.source_id != source_id: return "Source id doesn't match", 400 db_session.delete(notes) db_session.commit() return "OK", 200
def get_program(program_id=None): user = get_user() if not permitted(user, 'get', 'program', program_id): return "Not authorized", 401 program = db_session.query(MonitoringProgram).get(program_id) if program_id else None if program == None: return "Not found", 404 result = program_to_json(program) result['can_delete'] = permitted(user, 'delete', 'program', program_id) result['can_manage_managers'] = permitted(user, 'manage_managers', 'program', program_id) return jsonify(result), 200
def update_source_processing_notes(source_id=None, note_id=None): user = get_user() if not permitted(user, 'update', 'source', source_id): return "Not authorized", 401 notes = db_session.query(DataProcessingNotes).get(note_id) if notes.source_id != source_id: return "Source id doesn't match", 400 body = request.json # TODO: validate json notes.notes = body['notes'] db_session.add(notes) db_session.commit() return "OK", 201
def get_programs(user_id=None):
user = get_user()
if not permitted(user, 'list_programs', 'user', user_id):
return "Not authorized", 401
if user_id == None:
return "Not found", 404
rows = db_session.execute(
"""
SELECT monitoring_program.id, description
FROM monitoring_program, user_program_manager
WHERE monitoring_program.id = monitoring_program_id
AND user_id = :user_id""", {"user_id": user_id})
return jsonify_rows(rows)
def get_program_managers(program_id=None):
user = get_user()
if not permitted(user, 'list_managers', 'program', program_id):
return "Not authorized", 401
if program_id == None:
return "Not found", 404
rows = db_session.execute(
"""
SELECT user.id, email, first_name, last_name
FROM user, user_program_manager
WHERE user.id = user_id
AND monitoring_program_id = :program_id""", {"program_id": program_id})
return jsonify_rows(rows)
def post_import():
user = get_user()
body = request.json
try:
source_id = body['source_id']
except KeyError:
return jsonify('source_id is required'), 400
if not permitted(user, 'update', 'source', source_id):
return 'Not authorized', 401
# Check upload parameter
if 'upload_uuid' not in body:
return jsonify("upload_uuid is required"), 400
upload_uuid = body['upload_uuid']
file_path = get_upload_path(upload_uuid)
if not os.path.exists(file_path):
return jsonify("invalid upload_uuid"), 400
# Create new working directory for the import
data_import = DataImport(
source_id = source_id,
status_id = 1,
upload_uuid = body['upload_uuid'],
filename = get_upload_name(upload_uuid),
data_type = body.get('data_type'),
user_id = user.id
)
db_session.add(data_import)
db_session.commit()
import_id = data_import.id
# working_path = os.path.join(imports_path, "%04d" % import_id)
# working_path, import_id = next_path(os.path.join(imports_path, "%04d"))
# os.makedirs(working_path)
process_import_async(import_id, 'checking')
# TODO - Ideally should return 201 Created and URL of new resource
return jsonify(data_import_json(load_import(import_id)))
def create_source_processing_notes(source_id=None): user = get_user() if not permitted(user, 'update', 'source', source_id): return "Not authorized", 401 body = request.json # TODO: validate json notes = DataProcessingNotes() notes.notes = body['notes'] notes.user_id = user.id notes.source_id = source_id db_session.add(notes) db_session.commit() return "OK", 201
def get_source(source_id=None): user = get_user() if not permitted(user, 'get', 'source', source_id): return "Not authorized", 401 source = db_session.query(Source).get(source_id) if source_id else None if source == None: return "Not found", 404 result = source_to_json(source) result['can_delete'] = permitted(user, 'delete', 'source', source_id) result['can_import_data'] = permitted(user, 'import_data', 'source', source_id) result['can_manage_custodians'] = permitted(user, 'manage_custodians', 'source', source_id) return jsonify(result), 200
def update_user_role(user_id):
user = get_user()
if 'Administrator' not in get_roles(user):
return "Forbidden", 403
body = request.json
try:
new_role = body['role']
except KeyError:
return "Missing role", 400
db_session.execute("DELETE FROM user_role WHERE user_id = :user_id", { 'user_id': user_id })
print(new_role)
print(user_id)
db_session.execute("INSERT INTO user_role (user_id, role_id) SELECT :user_id, (SELECT id FROM role WHERE description = :role)", { 'user_id': user_id, 'role': new_role })
db_session.commit()
return "OK", 200
def get_sources():
user = get_user()
if not permitted(user, 'list', 'source'):
return "Not authorized", 401
db_session.execute("SET time_zone = '+00:00'")
program_id = request.args.get('program_id')
print(user.id)
rows = db_session.execute(
"""SELECT
source.id,
source.description,
data_import_status.code AS status,
source.time_created
FROM source
LEFT JOIN (SELECT source_id, max(data_import.id) AS data_import_id FROM data_import GROUP BY source_id) AS latest_import
ON latest_import.source_id = source.id
LEFT JOIN data_import ON latest_import.data_import_id = data_import.id
LEFT JOIN data_import_status ON data_import_status.id = data_import.status_id
WHERE
(
EXISTS (SELECT 1 FROM user_role WHERE user_id = :user_id AND role_id = 1) OR
(
EXISTS (SELECT 1 FROM user_role WHERE user_id = :user_id AND role_id = 2) AND
source.id IN (SELECT source_id FROM user_source WHERE user_id = :user_id)
) OR
(
EXISTS (SELECT 1 FROM user_role WHERE user_id = :user_id AND role_id = 3) AND
source.monitoring_program_id IN (SELECT monitoring_program_id FROM user_program_manager WHERE user_id = :user_id)
)
)
AND (:program_id IS NULL OR monitoring_program_id = :program_id)
""",
{ 'user_id': user.id, 'program_id': program_id })
return jsonify_rows(rows)
def users(): user = get_user() if 'Administrator' not in get_roles(user): return "Forbidden", 403 sql = """SELECT user.id, user.email, user.first_name, user.last_name, user.phone_number, MAX(role.description = 'Administrator') AS is_admin, COALESCE(MAX(role.description), 'Custodian') AS role FROM user LEFT JOIN user_role ON user.id = user_role.user_id LEFT JOIN role ON user_role.role_id = role.id GROUP BY user.id""" rows = db_session.execute(sql) return jsonify([dict(row) for row in rows])
def get_source_custodians(source_id=None):
user = get_user()
if not permitted(user, 'manage_custodians', 'source', source_id):
return "Not authorized", 401
source = db_session.query(Source).get(source_id) if source_id else None
if source == None:
return "Not found", 404
rows = db_session.execute("""SELECT
user.first_name,
user.last_name,
user.email,
user.id
FROM user_source
JOIN user ON user_source.user_id = user.id
WHERE user_source.source_id = :source_id
""", { 'source_id': source_id })
return jsonify_rows(rows)
def create_or_update_source(source_id=None):
action = 'update' if source_id else 'create'
user = get_user()
if not permitted(user, action, 'source', source_id):
return "Not authorized", 401
if source_id:
source = db_session.query(Source).get(source_id)
else:
source = Source()
if not source:
return "Not found", 400
body = request.json
errors = validate_fields(source_fields, body)
if len(errors):
return jsonify(errors), 400
update_source_from_json(source, body)
db_session.add(source)
db_session.flush()
if action == 'create':
db_session.execute("""INSERT INTO user_source (user_id, source_id) VALUES (:user_id, :source_id)""",
{ 'source_id': source.id, 'user_id': user.id })
else:
remove_orphaned_monitoring_programs()
db_session.commit()
return jsonify(source_to_json(source)), 200 if source_id else 201
def update_programs(user_id=None):
user = get_user()
if not permitted(user, 'update_programs', 'user', user_id):
return "Not authorized", 401
if user_id == None:
return "Not found", 404
body = request.json
db_session.execute(
"""DELETE FROM user_program_manager WHERE user_id = :user_id""",
{"user_id": user_id})
for program_id in body:
db_session.execute(
"""INSERT INTO user_program_manager (user_id, monitoring_program_id) VALUES (:user_id, :program_id)""",
{
"user_id": user_id,
"program_id": program_id
})
db_session.commit()
return "OK", 201
def create_source_custodian(source_id=None):
user = get_user()
if not permitted(user, 'manage_custodians', 'source', source_id):
return "Not authorized", 401
body = request.json
email = body["email"]
if not re.match(email_regex, email):
return jsonify({ 'error' : '"%s" is not a valid email address' % email }), 400
custodian = db_session.query(User).filter(User.email == email).one_or_none()
if not custodian:
if auto_create_custodians:
custodian = User(email=email)
db_session.add(custodian)
db_session.flush()
else:
error_message = 'No user found with the email address "%s". (Note: custodians must first create an account before they can be added)' % email
return jsonify({ 'error': error_message }), 400
rows = db_session.execute("""SELECT 1
FROM user_source
WHERE user_id = :user_id
AND source_id = :source_id
""", { 'source_id': source_id, 'user_id': custodian.id })
if len(list(rows)) == 0:
db_session.execute("""INSERT INTO user_source (user_id, source_id) VALUES (:user_id, :source_id)""",
{ 'source_id': source_id, 'user_id': custodian.id })
db_session.commit()
return "OK", 201
def approve_import(import_id=None):
data_import = load_import(import_id)
if not data_import:
return "Not found", 404
old_status = status_codes[data_import.status_id]
if old_status != 'imported':
return "Cannot approve import with status '%s'" % old_status, 400
user = get_user()
if not permitted(user, 'approve', 'import', import_id):
return 'Not authorized', 401
db_session.execute("UPDATE data_import SET status_id = :status_id WHERE id = :import_id", {
'status_id': status_ids['approved'],
'import_id': import_id
})
db_session.commit()
data_import.status_id = status_ids['approved']
return jsonify(data_import_json(data_import))
def current_user(): user = get_user() if user is None: return "Not found", 404 else: return jsonify(user_to_json(user)), 200
def is_logged_in(): return jsonify(get_user() is not None), 200
