def manage_apikeys(): if request.method == 'POST': rdata = {'status': 'success', 'message': '', } name = request.form['name'].strip() organization_id = request.form['organization'].strip() if not name: rdata['message'] = "Name is required" rdata['status'] = 'error' else: apikey = ApiKey(name) apikey.organization = Organization.query.get(organization_id) db.session.add(apikey) db.session.commit() data = [apikey.serialize] socketio.emit('manage-apikeys', {'data': data, 'action': 'add'}, namespace='/manage/apikeys', room='organization-' + str(apikey.organization_id) ) rdata['message'] = "Api key {} was successfully created in {}".format(apikey.name, apikey.organization.name) rdata['success'] = 'success' return jsonify(rdata) return render_template('manage/apikeys.html')
def post(self): rdata = {'success': False, 'message': "" } client_data = request.args data = request.json url_error = ScraperUrlError() url_error.run_uuid = client_data['scraperRun'] url_error.num_tries = data.get('numTries') url_error.reason = data.get('reason') url_error.ref_id = data.get('ref_id') url_error.ref_table = data.get('ref_table') url_error.status_code = data.get('statusCode') url_error.thread_name = data.get('threadName') url_error.url = data.get('url') db.session.add(url_error) db.session.commit() data = [{'rowId': client_data['scraperRun'], 'urlErrorCount': 1 }] socketio.emit('data-scrapers', {'data': data, 'action': 'increment'}, namespace='/data/scrapers/{env}/'.format(env=client_data['environment'].lower()), room='organization-{org_id}'.format(org_id=url_error.scraper_run.scraper.group.organization_id) ) rdata['success'] = True rdata['message'] = "" return rdata
def post(self): rdata = {'success': False, 'message': "" } client_data = request.args data = request.json group_id = Scraper.query.filter_by(key=client_data['scraperKey']).scalar().group_id run = ScraperRun() run.scraper_key = client_data['scraperKey'] run.group_id = group_id run.environment = client_data.get('environment') run.uuid = client_data['scraperRun'] run.start_time = data['startTime'] db.session.add(run) db.session.commit() data = [run.serialize] socketio.emit('data-scrapers', {'data': data, 'action': 'start'}, namespace='/data/scrapers/{env}/'.format(env=client_data['environment'].lower()), room='organization-{org_id}'.format(org_id=run.scraper.group.organization_id) ) rdata['success'] = True rdata['message'] = "" return rdata
def manage_group_edit(): rdata = {'success': False, 'message': "", } group_id = request.form['pk'] # group_field = request.form['name'].strip() # Not currently used group_new_name = request.form['value'].strip() # Check if the user has permission to update this value group = None try: group = Group.query.get(int(group_id)) if current_user not in group.organization.users or group is None: rdata['success'] = False rdata['message'] = "Invalid group" return jsonify(rdata) except: logger.exception("Error checking if user ({}) can access the group ({})" .format(current_user.id, group_id)) rdata['success'] = False rdata['message'] = "Invalid group" return jsonify(rdata) # Check if the new group name can be used current_groups = group.organization.groups group_list = [] # Create list of current groups to check if the name already exists for group_name in current_groups: group_list.append(group_name.name.strip().lower()) if group.name.lower() == 'default': # Can not rename the default group rdata['success'] = False rdata['message'] = "Cannot modify the Default group" return jsonify(rdata) elif (group_new_name.lower() == 'default' or group_new_name.lower() in group_list)\ and group_new_name.lower() != group.name.lower(): # There is always a default group, so this name cannot be used rdata['success'] = False rdata['message'] = "Group with name `{}` is already in use".format(group_new_name) return jsonify(rdata) # All error checking has been done, lets save the data group.name = group_new_name db.session.add(group) db.session.commit() logger.info("User {} updated group {} in {}" .format(current_user.email, group.name, group.organization.name)) data = [group.serialize] socketio.emit('manage-groups', {'data': data, 'action': 'update'}, namespace='/manage/groups', room='organization-' + str(group.organization_id) ) rdata['success'] = True return jsonify(rdata)
def post(self): rdata = {'success': False, 'message': "" } client_data = request.args log_data = request.form log = ScraperLog() log.run_uuid = client_data['scraperRun'] log.exc_info = log_data['exc_info'] log.exc_text = log_data['exc_text'] log.filename = log_data['filename'] log.func_name = log_data['funcName'] log.level_name = log_data['levelname'] log.level_no = log_data['levelno'] log.line_no = log_data['lineno'] log.message = log_data['message'] log.module = log_data['module'] log.name = log_data['name'] log.pathname = log_data['pathname'] log.process = log_data['process'] log.process_name = log_data['processName'] log.relative_created = log_data['relativeCreated'] log.stack_info = log_data['stack_info'] log.thread = log_data['thread'] log.thread_name = log_data['threadName'] db.session.add(log) db.session.commit() data = [{'rowId': client_data['scraperRun'], 'scraperKey': client_data['scraperKey'], 'groupId': log.scraper_run.scraper.group.id, }] if log.level_name == 'CRITICAL': data[0]['criticalCount'] = 1 if log.level_name == 'ERROR': data[0]['errorCount'] = 1 if log.level_name == 'WARNING': data[0]['warningCount'] = 1 socketio.emit('data-scrapers', {'data': data, 'action': 'increment'}, namespace='/data/scrapers/{env}/'.format(env=client_data['environment'].lower()), room='organization-{org_id}'.format(org_id=log.scraper_run.scraper.group.organization_id) ) rdata['success'] = True rdata['message'] = "" return rdata
def manage_apikey_edit(): rdata = {'success': False, 'message': "", } apikey_id = request.form['pk'] # apikey_field = request.form['name'].strip() # Not currently used apikey_new_name = request.form['value'].strip() # Check if the user has permission to update this value apikey = None try: apikey = ApiKey.query.get(int(apikey_id)) if current_user not in apikey.organization.users or apikey is None: rdata['success'] = False rdata['message'] = "Invalid apikey" return jsonify(rdata) except Exception: logger.exception("Error checking if user ({}) can access the apikey ({})" .format(current_user.id, apikey_id)) rdata['success'] = False rdata['message'] = "Invalid action" return jsonify(rdata) # Check if the new apikey name can be used current_apikey_names = apikey.organization.apikeys apikey_list = [] # Create list of current apikey name to check if the name already exists for key in current_apikey_names: apikey_list.append(key.name.strip().lower()) if apikey_new_name.lower() in apikey_list and apikey_new_name.lower() != apikey.name.lower(): rdata['success'] = False rdata['message'] = "Apikey with name `{}` is already in use".format(apikey_new_name) return jsonify(rdata) # All error checking has been done, lets save the data apikey.name = apikey_new_name db.session.add(apikey) db.session.commit() data = [apikey.serialize] socketio.emit('manage-apikeys', {'data': data, 'action': 'update'}, namespace='/manage/apikeys', room='organization-' + str(apikey.organization.id) ) rdata['success'] = True return jsonify(rdata)
def manage_group_delete(group_id): rdata = {'status': 'error', 'message': '', } group = Group.query.get(group_id) # Check if user can delete group if current_user not in group.organization.users: abort(403) if group.name.lower() == 'default': rdata['message'] = "Cannot delete default group" else: # If the group has scrapers, move them to the default group group_scrapers = list(group.scrapers) moved_scrapers_message = "" if len(group_scrapers) > 0: default_group = Group.query.filter_by(organization_id=group.organization_id)\ .filter_by(name='Default').scalar() for scraper in group_scrapers: default_group.scrapers.append(scraper) db.session.add(default_group) moved_scrapers_message = "Moved {num_scrapers} scrapers to the Default group"\ .format(num_scrapers=len(group_scrapers)) db.session.delete(group) db.session.commit() rdata['message'] = "Deleted Group {group_name} from {org_name}. {moved_scrapers}"\ .format(group_name=group.name, org_name=group.organization.name, moved_scrapers=moved_scrapers_message) rdata['status'] = 'success' logger.info("User {email} deleted group {group_name} from {org_name}" .format(email=current_user.email, group_name=group.name, org_name=group.organization.name)) data = [group.serialize] socketio.emit('manage-groups', {'data': data, 'action': 'delete'}, namespace='/manage/groups', room='organization-{org_id}'.format(org_id=group.organization_id) ) return jsonify(rdata)
def manage_scraper_delete(scraper_id): scraper = Scraper.query.get(scraper_id) # Check if user can delete group if current_user not in scraper.group.organization.users: abort(403) db.session.delete(scraper) db.session.commit() logger.info("User {} deleted scraper {} - {}" .format(current_user.email, scraper.key, scraper.name)) data = [scraper.serialize] socketio.emit('manage-scrapers', {'data': data, 'action': 'delete'}, namespace='/manage/scrapers', room='organization-' + str(scraper.group.organization.id) ) return jsonify({'message': "Deleted Scraper " + scraper.name, 'status': 'success'})
def manage_apikey_delete(apikey_id): apikey = ApiKey.query.get(apikey_id) # Check if user can delete apikey if current_user not in apikey.organization.users: abort(403) db.session.delete(apikey) db.session.commit() logger.info("User {} deleted API Key {} from {}".format(current_user, apikey.name, apikey.organization.name)) data = [apikey.serialize] socketio.emit('manage-apikeys', {'data': data, 'action': 'delete'}, namespace='/manage/apikeys', room='organization-' + str(apikey.organization.id) ) return jsonify({'message': "Deleted API key {} from {}" .format(apikey.name, apikey.organization.name), 'status': 'success', })
def manage_organizations(): if request.method == 'POST': rdata = {'status': False, 'message': '', } name = request.form['name'].strip() if not name: rdata['message'] = "Name is required" rdata['status'] = 'error' else: # Check if organization name already exists is_organization = Organization.query.filter_by(name=name).scalar() if is_organization is not None: rdata['message'] = "Organization with name {} already exists".format(name) rdata['status'] = 'error' else: organization = Organization(name=name, user=current_user) db.session.add(organization) db.session.flush() current_user.organizations.append(organization) group = Group(name='Default', organization_id=organization.id) db.session.add(group) db.session.commit() logger.info("User {} created organization {}".format(current_user.username, organization.name)) data = [organization.serialize] # Add current user to the new room # TODO: Figure how to join a room here # socketio.join_room('organization-' + str(organization.id)) socketio.emit('manage-organizations', {'data': data, 'action': 'add'}, namespace='/manage/organizations', room='organization-' + str(organization.id) ) rdata['message'] = "Organization {} was successfully created".format(organization.name) rdata['status'] = 'success' return jsonify(rdata) return render_template('manage/organizations.html')
def post(self): rdata = {'success': False, 'message': "" } client_data = request.args data = request.json run = ScraperRun.query.filter_by(uuid=client_data['scraperRun']).scalar() run.total_urls_hit = data.get('totalUrls') run.ref_data_count = data.get('refDataCount') run.ref_data_success_count = data.get('refDataSuccessCount') run.num_rows_added_to_db = data.get('rowsAddedToDb') run.stop_time = datetime.datetime.strptime(data['stopTime'], "%Y-%m-%d %H:%M:%S.%f") # Calc runtime and get counts runtime = run.stop_time - run.start_time run.runtime = runtime.total_seconds() counts = ScraperLog.query.filter_by(run_uuid=client_data['scraperRun']) run.critical_count = counts.filter_by(level_name='CRITICAL').count() run.error_count = counts.filter_by(level_name='ERROR').count() run.warning_count = counts.filter_by(level_name='WARNING').count() run.url_error_count = ScraperUrlError.query.filter_by(run_uuid=client_data['scraperRun']).count() db.session.commit() data = [run.serialize] socketio.emit('data-scrapers', {'data': data, 'action': 'stop'}, namespace='/data/scrapers/{env}/'.format(env=client_data['environment'].lower()), room='organization-{org_id}'.format(org_id=run.scraper.group.organization_id) ) rdata['success'] = True rdata['message'] = "" return rdata
def manage_organizations_delete(organization_id): rdata = {'status': 'error', 'message': '', } organization = Organization.query.get(organization_id) # Check if user can delete group if current_user.id != organization.owner_id: abort(403) # Check if organization has scrapers still assigned to it org_has_scrapers = False for group in organization.groups: if len(list(group.scrapers)) > 0: org_has_scrapers = True break print(org_has_scrapers) if organization.name == current_user.username: rdata['message'] = "Cannot delete your private organization" elif org_has_scrapers is True: rdata['message'] = "Cannot delete. Organization has scrapers under it" else: db.session.delete(organization) db.session.commit() logger.info("User {username} deleted organization {org_name}" .format(username=current_user.username, org_name=organization.name)) data = [organization.serialize] socketio.emit('manage-organizations', {'data': data, 'action': 'delete'}, namespace='/manage/organizations', room='organization-{org_id}'.format(org_id=organization.id) ) rdata['message'] = "Deleted organization {org_name}".format(org_name=organization.name) rdata['status'] = 'success' return jsonify(rdata)
def manage_groups(): if request.method == 'POST': rdata = {'status': 'success', 'message': '', } name = request.form['name'].strip() organization_id = request.form['organization'].strip() if not name: rdata['message'] = "Name is required" rdata['status'] = 'error' else: # Check if group name for organization already exists is_group = Group.query.filter_by(organization_id=organization_id).filter_by(name=name).scalar() if is_group is not None: rdata['message'] = "Group with name {} already exists".format(name) rdata['status'] = 'error' else: # TODO: Make sure current_user has perms to create a group in this organization group = Group(name=name, organization_id=organization_id) group.organization = Organization.query.get(organization_id) db.session.add(group) db.session.commit() logger.info("User {} created group {} in {}" .format(current_user.email, group.name, group.organization.name)) data = [group.serialize] socketio.emit('manage-groups', {'data': data, 'action': 'add'}, namespace='/manage/groups', room='organization-' + str(organization_id) ) rdata['message'] = "Group {} was successfully created in {}".format(group.name, group.organization.name) rdata['status'] = 'success' return jsonify(rdata) return render_template('manage/groups.html')
def manage_scraper_edit(): rdata = {'success': False, 'message': "", 'displayAlert': None, } scraper_id = request.form['pk'] scraper_field = request.form['name'].strip() scraper_new_value = request.form['value'].strip() # Check if the user has permission to update this value scraper = None try: scraper = Scraper.query.get(int(scraper_id)) scraper_users = scraper.group.organization.users if current_user not in scraper_users or scraper is None: rdata['success'] = False rdata['message'] = "Invalid scraper" return jsonify(rdata) except Exception: logger.exception("Error checking if user ({user_id}) can access the scraper ({scraper_id})" .format(user_id=current_user.id, scraper_id=scraper_id)) rdata['success'] = False rdata['message'] = "Invalid action" return jsonify(rdata) # Checks depending on the field being updated if scraper_field == 'name': # Scrapers can have the same name # All error checking has been done, lets save the data scraper.name = scraper_new_value elif scraper_field == 'group': try: new_group = Group.query.get(int(scraper_new_value)) # Make sure user has permissions for the group if current_user not in new_group.organization.users: rdata['success'] = False rdata['message'] = "Invalid group" return jsonify(rdata) # Check if the current owner is part of this new group, if not set the owner to current user if scraper.owner not in new_group.organization.users: rdata['displayAlert'] = {'status': 'warning', 'message': "User <b>{owner}</b> is not in in the selected " "organization <b>{org_name}</b>" "\nSetting scraper owner to <b>{current_user}</b>" .format(owner=scraper.owner.username, org_name=new_group.organization.name, current_user=current_user.username, ), } # Default owner to current user scraper.owner = current_user # Set the new group scraper.group = new_group except Exception: logger.exception("Error checking if user ({user_id}) can use the group ({new_val})" .format(user_id=current_user.id, new_val=scraper_new_value)) rdata['success'] = False rdata['message'] = "Invalid action" return jsonify(rdata) elif scraper_field == 'owner': if scraper_new_value is None or scraper_new_value == '': scraper.owner = None else: try: new_user = User.query.get(int(scraper_new_value)) if new_user not in scraper_users or current_user not in scraper_users: rdata['success'] = False rdata['message'] = "Invalid user" return jsonify(rdata) # Everything looks good scraper.owner = new_user except Exception: logger.exception("Error checking if user ({new_val}) is in the organization ({org_name})" .format(new_val=scraper_new_value, org_name=scraper.group.organization.name)) rdata['success'] = False rdata['message'] = "Invalid action" return jsonify(rdata) db.session.add(scraper) db.session.commit() rdata['success'] = True logger.info("User {email} updated scraper {scraper_key} - {scraper_name}" .format(email=current_user.email, scraper_key=scraper.key, scraper_name=scraper.name)) data = [scraper.serialize] socketio.emit('manage-scrapers', {'data': data, 'action': 'update'}, namespace='/manage/scrapers', room='organization-{org_id}'.format(org_id=scraper.group.organization_id) ) return jsonify(rdata)
def manage_scrapers(): if request.method == 'POST': rdata = {'status': 'success', 'message': "", } name = request.form['name'].strip() try: group = Group.query.get(int(request.form['group'].strip())) except ValueError: group = None try: owner = User.query.get(int(request.form['owner'].strip())) except ValueError: owner = None if not name: rdata['message'] = "Name is required" rdata['status'] = 'error' if group is None: rdata['message'] = "Group is required" rdata['status'] = 'error' elif owner is not None and owner not in group.organization.users: # If the user is not in the org that the scraper is in rdata['message'] = "Invalid owner for the scraper" rdata['status'] = 'error' else: scraper = Scraper(name) scraper.owner = owner scraper.group = group db.session.add(scraper) # Flush to get the id so it can be encoded db.session.flush() # Do not change the string in the function below. # It is just used to create the scraper key, nothing secure scraper.key = generate_key(scraper.id, 'Scraper Salt 123') db.session.add(scraper) db.session.commit() logger.info("User {} created scraper {} - {}" .format(current_user.email, scraper.key, scraper.name)) data = [scraper.serialize] socketio.emit('manage-scrapers', {'data': data, 'action': 'add'}, namespace='/manage/scrapers', room='organization-' + str(scraper.group.organization.id) ) rdata['message'] = "Scraper {} was successfully created in group {}"\ .format(scraper.name, scraper.group.name) rdata['status'] = 'success' return jsonify(rdata) group_list = [] for organization in current_user.organizations: groups = Group.query.filter_by(organization=organization).all() groups = [i.serialize for i in groups] group_list.extend(groups) return render_template('manage/scrapers.html', groups=group_list, )
def set(self, key, value): try: self.store[key] = value socketio.emit(key, value, broadcast=True) except KeyError: socketio.send('error', 'KeyError')