def get_source_imports(source_id=None): user = get_user() if not permitted(user, 'get', 'source', source_id): return "Not authorized", 401 source = db_session.query(Source).get(source_id) if source_id else None if source == None: return "Not found", 404 db_session.execute("SET time_zone = '+00:00'") rows = db_session.execute("""SELECT data_import.id, data_import.filename, data_import.data_type, data_import_status.code AS status, data_import.time_created, data_import.upload_uuid FROM data_import LEFT JOIN data_import_status ON data_import_status.id = data_import.status_id WHERE data_import.source_id = :source_id """, { 'source_id': source_id }) return jsonify_rows(rows)
def create_or_update_program(program_id=None): action = 'update' if program_id else 'create' user = get_user() if not permitted(user, action, 'program', program_id): return "Not authorized", 401 if program_id: program = db_session.query(MonitoringProgram).get(program_id) else: program = MonitoringProgram() if not program: return "Not found", 400 body = request.json errors = validate_fields(program_fields, body) if len(errors): return jsonify(errors), 400 update_program_from_json(program, body) db_session.add(program) db_session.flush() if action == 'create': db_session.execute("""INSERT INTO user_program_manager (user_id, monitoring_program_id) VALUES (:user_id, :program_id)""", { 'program_id': program.id, 'user_id': user.id }) db_session.commit() return jsonify(program_to_json(program)), 200 if program_id else 201
def get_source_processing_notes(source_id=None): user = get_user() if not permitted(user, 'get', 'source', source_id): return "Not authorized", 401 source = db_session.query(Source).get(source_id) if source_id else None if source == None: return "Not found", 404 db_session.execute("SET time_zone = '+00:00'") rows = db_session.execute("""SELECT notes.id, user.first_name, user.last_name, user.email, notes.source_id, notes.time_created, notes.notes, user.id = :user_id AS editable FROM data_processing_notes notes JOIN user ON notes.user_id = user.id WHERE notes.source_id = :source_id """, { 'source_id': source_id, 'user_id': user.id }) return jsonify_rows(rows)
def add_manager_to_monitoring_program(program_id = None): if program_id == None: return "Not found", 404 user = get_user() if not permitted(user, 'mangage_managers', 'program', program_id): return "Not authorized", 401 body = request.json email = body["email"] if not re.match(email_regex, email): return jsonify({ 'error' : '"%s" is not a valid email address' % email }), 400 manager = db_session.query(User).filter(User.email == email).one_or_none() if not manager: manager = User(email=email) db_session.add(manager) db_session.flush() db_session.execute("""REPLACE INTO user_program_manager (user_id, monitoring_program_id) VALUES (:user_id, :program_id)""", { 'user_id': manager.id, 'program_id': program_id }) db_session.commit() return "OK", 201
def delete_source_custodian(source_id=None, user_id=None): user = get_user() if not permitted(user, 'manage_custodians', 'source', source_id): return "Not authorized", 401 db_session.execute("""DELETE FROM user_source WHERE user_id = :user_id AND source_id = :source_id""", { 'source_id': source_id, 'user_id': user_id }) db_session.commit() return "OK", 200
def process_import_async(import_id, status): info = load_import(import_id) user = get_user() file_path = get_upload_path(info.upload_uuid) working_path = import_path(import_id) os.makedirs(working_path, exist_ok=True) data_type = info.data_type with lock: running_imports[import_id] = { 'started': local_iso_datetime(), 'status': status, 'total_rows': 0, 'processed_rows': 0, 'is_admin': 'Administrator' in get_roles(user) } def result_callback(result): is_admin = running_imports[import_id]['is_admin'] with lock: del running_imports[import_id] success = result['errors'] == 0 if status == 'checking': new_status = 'checked_ok' if success else 'checked_error' elif status == 'importing': new_status = ('approved' if is_admin else 'imported') if success else 'import_error' info = load_import(import_id) info.status_id = status_ids[new_status] info.error_count = result['errors'] info.warning_count = result['warnings'] db_session.commit() def progress_callback(processed_rows, total_rows): with lock: running_imports[import_id]['total_rows'] = total_rows running_imports[import_id]['processed_rows'] = processed_rows try: # Start import process t = Thread(target = process_import, args = (file_path, working_path, data_type, status == 'importing', progress_callback, result_callback, info.source_id, import_id)) t.start() except: traceback.print_exc() result_callback({ 'warnings': 0, 'errors': 1 })
def delete_source(source_id=None): user = get_user() if not permitted(user, 'delete', 'source', source_id): return "Not authorized", 401 db_session.execute("""DELETE FROM user_source WHERE source_id = :source_id""", { 'source_id': source_id }) db_session.execute("""DELETE FROM data_import WHERE source_id = :source_id""", { 'source_id': source_id }) db_session.execute("""DELETE FROM source WHERE id = :source_id""", { 'source_id': source_id }) remove_orphaned_monitoring_programs() db_session.commit() return "OK", 200
def remove_manager_from_monitoring_program(program_id = None, user_id = None): if program_id == None or user_id == None: return "Not found", 404 user = get_user() if not permitted(user, 'mangage_managers', 'program', program_id): return "Not authorized", 401 db_session.execute("""DELETE FROM user_program_manager WHERE user_id = :user_id AND monitoring_program_id = :program_id""", { 'program_id': program_id, 'user_id': user_id }) db_session.commit() return "OK", 200
def remove_source_from_monitoring_program(program_id = None, source_id = None): if program_id == None or source_id == None: return "Not found", 404 user = get_user() if not permitted(user, 'update', 'source', program_id): return "Not authorized", 401 db_session.execute("""UPDATE source SET monitoring_program_id = NULL WHERE id = :source_id AND monitoring_program_id = :program_id""", { 'program_id': program_id, 'source_id': source_id }) db_session.commit() return "OK", 200
def delete_monitoring_program(program_id = None): if program_id == None: return "Not found", 404 user = get_user() if not permitted(user, 'delete', 'program', program_id): return "Not authorized", 401 db_session.execute("""DELETE FROM monitoring_program WHERE id = :program_id""", { 'program_id': program_id }) db_session.commit() return "OK", 200
def delete_source_processing_notes(source_id=None, note_id=None): user = get_user() if not permitted(user, 'delete', 'source', source_id): return "Not authorized", 401 notes = db_session.query(DataProcessingNotes).get(note_id) if notes.source_id != source_id: return "Source id doesn't match", 400 db_session.delete(notes) db_session.commit() return "OK", 200
def get_program(program_id=None): user = get_user() if not permitted(user, 'get', 'program', program_id): return "Not authorized", 401 program = db_session.query(MonitoringProgram).get(program_id) if program_id else None if program == None: return "Not found", 404 result = program_to_json(program) result['can_delete'] = permitted(user, 'delete', 'program', program_id) result['can_manage_managers'] = permitted(user, 'manage_managers', 'program', program_id) return jsonify(result), 200
def update_source_processing_notes(source_id=None, note_id=None): user = get_user() if not permitted(user, 'update', 'source', source_id): return "Not authorized", 401 notes = db_session.query(DataProcessingNotes).get(note_id) if notes.source_id != source_id: return "Source id doesn't match", 400 body = request.json # TODO: validate json notes.notes = body['notes'] db_session.add(notes) db_session.commit() return "OK", 201
def get_programs(user_id=None): user = get_user() if not permitted(user, 'list_programs', 'user', user_id): return "Not authorized", 401 if user_id == None: return "Not found", 404 rows = db_session.execute( """ SELECT monitoring_program.id, description FROM monitoring_program, user_program_manager WHERE monitoring_program.id = monitoring_program_id AND user_id = :user_id""", {"user_id": user_id}) return jsonify_rows(rows)
def get_program_managers(program_id=None): user = get_user() if not permitted(user, 'list_managers', 'program', program_id): return "Not authorized", 401 if program_id == None: return "Not found", 404 rows = db_session.execute( """ SELECT user.id, email, first_name, last_name FROM user, user_program_manager WHERE user.id = user_id AND monitoring_program_id = :program_id""", {"program_id": program_id}) return jsonify_rows(rows)
def post_import(): user = get_user() body = request.json try: source_id = body['source_id'] except KeyError: return jsonify('source_id is required'), 400 if not permitted(user, 'update', 'source', source_id): return 'Not authorized', 401 # Check upload parameter if 'upload_uuid' not in body: return jsonify("upload_uuid is required"), 400 upload_uuid = body['upload_uuid'] file_path = get_upload_path(upload_uuid) if not os.path.exists(file_path): return jsonify("invalid upload_uuid"), 400 # Create new working directory for the import data_import = DataImport( source_id = source_id, status_id = 1, upload_uuid = body['upload_uuid'], filename = get_upload_name(upload_uuid), data_type = body.get('data_type'), user_id = user.id ) db_session.add(data_import) db_session.commit() import_id = data_import.id # working_path = os.path.join(imports_path, "%04d" % import_id) # working_path, import_id = next_path(os.path.join(imports_path, "%04d")) # os.makedirs(working_path) process_import_async(import_id, 'checking') # TODO - Ideally should return 201 Created and URL of new resource return jsonify(data_import_json(load_import(import_id)))
def create_source_processing_notes(source_id=None): user = get_user() if not permitted(user, 'update', 'source', source_id): return "Not authorized", 401 body = request.json # TODO: validate json notes = DataProcessingNotes() notes.notes = body['notes'] notes.user_id = user.id notes.source_id = source_id db_session.add(notes) db_session.commit() return "OK", 201
def get_source(source_id=None): user = get_user() if not permitted(user, 'get', 'source', source_id): return "Not authorized", 401 source = db_session.query(Source).get(source_id) if source_id else None if source == None: return "Not found", 404 result = source_to_json(source) result['can_delete'] = permitted(user, 'delete', 'source', source_id) result['can_import_data'] = permitted(user, 'import_data', 'source', source_id) result['can_manage_custodians'] = permitted(user, 'manage_custodians', 'source', source_id) return jsonify(result), 200
def update_user_role(user_id): user = get_user() if 'Administrator' not in get_roles(user): return "Forbidden", 403 body = request.json try: new_role = body['role'] except KeyError: return "Missing role", 400 db_session.execute("DELETE FROM user_role WHERE user_id = :user_id", { 'user_id': user_id }) print(new_role) print(user_id) db_session.execute("INSERT INTO user_role (user_id, role_id) SELECT :user_id, (SELECT id FROM role WHERE description = :role)", { 'user_id': user_id, 'role': new_role }) db_session.commit() return "OK", 200
def get_sources(): user = get_user() if not permitted(user, 'list', 'source'): return "Not authorized", 401 db_session.execute("SET time_zone = '+00:00'") program_id = request.args.get('program_id') print(user.id) rows = db_session.execute( """SELECT source.id, source.description, data_import_status.code AS status, source.time_created FROM source LEFT JOIN (SELECT source_id, max(data_import.id) AS data_import_id FROM data_import GROUP BY source_id) AS latest_import ON latest_import.source_id = source.id LEFT JOIN data_import ON latest_import.data_import_id = data_import.id LEFT JOIN data_import_status ON data_import_status.id = data_import.status_id WHERE ( EXISTS (SELECT 1 FROM user_role WHERE user_id = :user_id AND role_id = 1) OR ( EXISTS (SELECT 1 FROM user_role WHERE user_id = :user_id AND role_id = 2) AND source.id IN (SELECT source_id FROM user_source WHERE user_id = :user_id) ) OR ( EXISTS (SELECT 1 FROM user_role WHERE user_id = :user_id AND role_id = 3) AND source.monitoring_program_id IN (SELECT monitoring_program_id FROM user_program_manager WHERE user_id = :user_id) ) ) AND (:program_id IS NULL OR monitoring_program_id = :program_id) """, { 'user_id': user.id, 'program_id': program_id }) return jsonify_rows(rows)
def users(): user = get_user() if 'Administrator' not in get_roles(user): return "Forbidden", 403 sql = """SELECT user.id, user.email, user.first_name, user.last_name, user.phone_number, MAX(role.description = 'Administrator') AS is_admin, COALESCE(MAX(role.description), 'Custodian') AS role FROM user LEFT JOIN user_role ON user.id = user_role.user_id LEFT JOIN role ON user_role.role_id = role.id GROUP BY user.id""" rows = db_session.execute(sql) return jsonify([dict(row) for row in rows])
def get_source_custodians(source_id=None): user = get_user() if not permitted(user, 'manage_custodians', 'source', source_id): return "Not authorized", 401 source = db_session.query(Source).get(source_id) if source_id else None if source == None: return "Not found", 404 rows = db_session.execute("""SELECT user.first_name, user.last_name, user.email, user.id FROM user_source JOIN user ON user_source.user_id = user.id WHERE user_source.source_id = :source_id """, { 'source_id': source_id }) return jsonify_rows(rows)
def create_or_update_source(source_id=None): action = 'update' if source_id else 'create' user = get_user() if not permitted(user, action, 'source', source_id): return "Not authorized", 401 if source_id: source = db_session.query(Source).get(source_id) else: source = Source() if not source: return "Not found", 400 body = request.json errors = validate_fields(source_fields, body) if len(errors): return jsonify(errors), 400 update_source_from_json(source, body) db_session.add(source) db_session.flush() if action == 'create': db_session.execute("""INSERT INTO user_source (user_id, source_id) VALUES (:user_id, :source_id)""", { 'source_id': source.id, 'user_id': user.id }) else: remove_orphaned_monitoring_programs() db_session.commit() return jsonify(source_to_json(source)), 200 if source_id else 201
def update_programs(user_id=None): user = get_user() if not permitted(user, 'update_programs', 'user', user_id): return "Not authorized", 401 if user_id == None: return "Not found", 404 body = request.json db_session.execute( """DELETE FROM user_program_manager WHERE user_id = :user_id""", {"user_id": user_id}) for program_id in body: db_session.execute( """INSERT INTO user_program_manager (user_id, monitoring_program_id) VALUES (:user_id, :program_id)""", { "user_id": user_id, "program_id": program_id }) db_session.commit() return "OK", 201
def create_source_custodian(source_id=None): user = get_user() if not permitted(user, 'manage_custodians', 'source', source_id): return "Not authorized", 401 body = request.json email = body["email"] if not re.match(email_regex, email): return jsonify({ 'error' : '"%s" is not a valid email address' % email }), 400 custodian = db_session.query(User).filter(User.email == email).one_or_none() if not custodian: if auto_create_custodians: custodian = User(email=email) db_session.add(custodian) db_session.flush() else: error_message = 'No user found with the email address "%s". (Note: custodians must first create an account before they can be added)' % email return jsonify({ 'error': error_message }), 400 rows = db_session.execute("""SELECT 1 FROM user_source WHERE user_id = :user_id AND source_id = :source_id """, { 'source_id': source_id, 'user_id': custodian.id }) if len(list(rows)) == 0: db_session.execute("""INSERT INTO user_source (user_id, source_id) VALUES (:user_id, :source_id)""", { 'source_id': source_id, 'user_id': custodian.id }) db_session.commit() return "OK", 201
def approve_import(import_id=None): data_import = load_import(import_id) if not data_import: return "Not found", 404 old_status = status_codes[data_import.status_id] if old_status != 'imported': return "Cannot approve import with status '%s'" % old_status, 400 user = get_user() if not permitted(user, 'approve', 'import', import_id): return 'Not authorized', 401 db_session.execute("UPDATE data_import SET status_id = :status_id WHERE id = :import_id", { 'status_id': status_ids['approved'], 'import_id': import_id }) db_session.commit() data_import.status_id = status_ids['approved'] return jsonify(data_import_json(data_import))
def current_user(): user = get_user() if user is None: return "Not found", 404 else: return jsonify(user_to_json(user)), 200
def is_logged_in(): return jsonify(get_user() is not None), 200