def manage_apikeys():
if request.method == 'POST':
rdata = {'status': 'success',
'message': '',
}
name = request.form['name'].strip()
organization_id = request.form['organization'].strip()
if not name:
rdata['message'] = "Name is required"
rdata['status'] = 'error'
else:
apikey = ApiKey(name)
apikey.organization = Organization.query.get(organization_id)
db.session.add(apikey)
db.session.commit()
data = [apikey.serialize]
socketio.emit('manage-apikeys',
{'data': data, 'action': 'add'},
namespace='/manage/apikeys',
room='organization-' + str(apikey.organization_id)
)
rdata['message'] = "Api key {} was successfully created in {}".format(apikey.name,
apikey.organization.name)
rdata['success'] = 'success'
return jsonify(rdata)
return render_template('manage/apikeys.html')
def post(self):
rdata = {'success': False,
'message': ""
}
client_data = request.args
data = request.json
url_error = ScraperUrlError()
url_error.run_uuid = client_data['scraperRun']
url_error.num_tries = data.get('numTries')
url_error.reason = data.get('reason')
url_error.ref_id = data.get('ref_id')
url_error.ref_table = data.get('ref_table')
url_error.status_code = data.get('statusCode')
url_error.thread_name = data.get('threadName')
url_error.url = data.get('url')
db.session.add(url_error)
db.session.commit()
data = [{'rowId': client_data['scraperRun'],
'urlErrorCount': 1
}]
socketio.emit('data-scrapers',
{'data': data, 'action': 'increment'},
namespace='/data/scrapers/{env}/'.format(env=client_data['environment'].lower()),
room='organization-{org_id}'.format(org_id=url_error.scraper_run.scraper.group.organization_id)
)
rdata['success'] = True
rdata['message'] = ""
return rdata
def post(self):
rdata = {'success': False,
'message': ""
}
client_data = request.args
data = request.json
group_id = Scraper.query.filter_by(key=client_data['scraperKey']).scalar().group_id
run = ScraperRun()
run.scraper_key = client_data['scraperKey']
run.group_id = group_id
run.environment = client_data.get('environment')
run.uuid = client_data['scraperRun']
run.start_time = data['startTime']
db.session.add(run)
db.session.commit()
data = [run.serialize]
socketio.emit('data-scrapers',
{'data': data, 'action': 'start'},
namespace='/data/scrapers/{env}/'.format(env=client_data['environment'].lower()),
room='organization-{org_id}'.format(org_id=run.scraper.group.organization_id)
)
rdata['success'] = True
rdata['message'] = ""
return rdata
def manage_group_edit():
rdata = {'success': False,
'message': "",
}
group_id = request.form['pk']
# group_field = request.form['name'].strip() # Not currently used
group_new_name = request.form['value'].strip()
# Check if the user has permission to update this value
group = None
try:
group = Group.query.get(int(group_id))
if current_user not in group.organization.users or group is None:
rdata['success'] = False
rdata['message'] = "Invalid group"
return jsonify(rdata)
except:
logger.exception("Error checking if user ({}) can access the group ({})"
.format(current_user.id, group_id))
rdata['success'] = False
rdata['message'] = "Invalid group"
return jsonify(rdata)
# Check if the new group name can be used
current_groups = group.organization.groups
group_list = []
# Create list of current groups to check if the name already exists
for group_name in current_groups:
group_list.append(group_name.name.strip().lower())
if group.name.lower() == 'default':
# Can not rename the default group
rdata['success'] = False
rdata['message'] = "Cannot modify the Default group"
return jsonify(rdata)
elif (group_new_name.lower() == 'default' or group_new_name.lower() in group_list)\
and group_new_name.lower() != group.name.lower():
# There is always a default group, so this name cannot be used
rdata['success'] = False
rdata['message'] = "Group with name `{}` is already in use".format(group_new_name)
return jsonify(rdata)
# All error checking has been done, lets save the data
group.name = group_new_name
db.session.add(group)
db.session.commit()
logger.info("User {} updated group {} in {}"
.format(current_user.email, group.name, group.organization.name))
data = [group.serialize]
socketio.emit('manage-groups',
{'data': data, 'action': 'update'},
namespace='/manage/groups',
room='organization-' + str(group.organization_id)
)
rdata['success'] = True
return jsonify(rdata)
def post(self):
rdata = {'success': False,
'message': ""
}
client_data = request.args
log_data = request.form
log = ScraperLog()
log.run_uuid = client_data['scraperRun']
log.exc_info = log_data['exc_info']
log.exc_text = log_data['exc_text']
log.filename = log_data['filename']
log.func_name = log_data['funcName']
log.level_name = log_data['levelname']
log.level_no = log_data['levelno']
log.line_no = log_data['lineno']
log.message = log_data['message']
log.module = log_data['module']
log.name = log_data['name']
log.pathname = log_data['pathname']
log.process = log_data['process']
log.process_name = log_data['processName']
log.relative_created = log_data['relativeCreated']
log.stack_info = log_data['stack_info']
log.thread = log_data['thread']
log.thread_name = log_data['threadName']
db.session.add(log)
db.session.commit()
data = [{'rowId': client_data['scraperRun'],
'scraperKey': client_data['scraperKey'],
'groupId': log.scraper_run.scraper.group.id,
}]
if log.level_name == 'CRITICAL':
data[0]['criticalCount'] = 1
if log.level_name == 'ERROR':
data[0]['errorCount'] = 1
if log.level_name == 'WARNING':
data[0]['warningCount'] = 1
socketio.emit('data-scrapers',
{'data': data, 'action': 'increment'},
namespace='/data/scrapers/{env}/'.format(env=client_data['environment'].lower()),
room='organization-{org_id}'.format(org_id=log.scraper_run.scraper.group.organization_id)
)
rdata['success'] = True
rdata['message'] = ""
return rdata
def manage_apikey_edit():
rdata = {'success': False,
'message': "",
}
apikey_id = request.form['pk']
# apikey_field = request.form['name'].strip() # Not currently used
apikey_new_name = request.form['value'].strip()
# Check if the user has permission to update this value
apikey = None
try:
apikey = ApiKey.query.get(int(apikey_id))
if current_user not in apikey.organization.users or apikey is None:
rdata['success'] = False
rdata['message'] = "Invalid apikey"
return jsonify(rdata)
except Exception:
logger.exception("Error checking if user ({}) can access the apikey ({})"
.format(current_user.id, apikey_id))
rdata['success'] = False
rdata['message'] = "Invalid action"
return jsonify(rdata)
# Check if the new apikey name can be used
current_apikey_names = apikey.organization.apikeys
apikey_list = []
# Create list of current apikey name to check if the name already exists
for key in current_apikey_names:
apikey_list.append(key.name.strip().lower())
if apikey_new_name.lower() in apikey_list and apikey_new_name.lower() != apikey.name.lower():
rdata['success'] = False
rdata['message'] = "Apikey with name `{}` is already in use".format(apikey_new_name)
return jsonify(rdata)
# All error checking has been done, lets save the data
apikey.name = apikey_new_name
db.session.add(apikey)
db.session.commit()
data = [apikey.serialize]
socketio.emit('manage-apikeys',
{'data': data, 'action': 'update'},
namespace='/manage/apikeys',
room='organization-' + str(apikey.organization.id)
)
rdata['success'] = True
return jsonify(rdata)
def manage_group_delete(group_id):
rdata = {'status': 'error',
'message': '',
}
group = Group.query.get(group_id)
# Check if user can delete group
if current_user not in group.organization.users:
abort(403)
if group.name.lower() == 'default':
rdata['message'] = "Cannot delete default group"
else:
# If the group has scrapers, move them to the default group
group_scrapers = list(group.scrapers)
moved_scrapers_message = ""
if len(group_scrapers) > 0:
default_group = Group.query.filter_by(organization_id=group.organization_id)\
.filter_by(name='Default').scalar()
for scraper in group_scrapers:
default_group.scrapers.append(scraper)
db.session.add(default_group)
moved_scrapers_message = "Moved {num_scrapers} scrapers to the Default group"\
.format(num_scrapers=len(group_scrapers))
db.session.delete(group)
db.session.commit()
rdata['message'] = "Deleted Group {group_name} from {org_name}. {moved_scrapers}"\
.format(group_name=group.name,
org_name=group.organization.name,
moved_scrapers=moved_scrapers_message)
rdata['status'] = 'success'
logger.info("User {email} deleted group {group_name} from {org_name}"
.format(email=current_user.email, group_name=group.name, org_name=group.organization.name))
data = [group.serialize]
socketio.emit('manage-groups',
{'data': data, 'action': 'delete'},
namespace='/manage/groups',
room='organization-{org_id}'.format(org_id=group.organization_id)
)
return jsonify(rdata)
def manage_scraper_delete(scraper_id):
scraper = Scraper.query.get(scraper_id)
# Check if user can delete group
if current_user not in scraper.group.organization.users:
abort(403)
db.session.delete(scraper)
db.session.commit()
logger.info("User {} deleted scraper {} - {}"
.format(current_user.email, scraper.key, scraper.name))
data = [scraper.serialize]
socketio.emit('manage-scrapers',
{'data': data, 'action': 'delete'},
namespace='/manage/scrapers',
room='organization-' + str(scraper.group.organization.id)
)
return jsonify({'message': "Deleted Scraper " + scraper.name, 'status': 'success'})
def manage_apikey_delete(apikey_id):
apikey = ApiKey.query.get(apikey_id)
# Check if user can delete apikey
if current_user not in apikey.organization.users:
abort(403)
db.session.delete(apikey)
db.session.commit()
logger.info("User {} deleted API Key {} from {}".format(current_user,
apikey.name,
apikey.organization.name))
data = [apikey.serialize]
socketio.emit('manage-apikeys',
{'data': data, 'action': 'delete'},
namespace='/manage/apikeys',
room='organization-' + str(apikey.organization.id)
)
return jsonify({'message': "Deleted API key {} from {}"
.format(apikey.name, apikey.organization.name),
'status': 'success',
})
def manage_organizations():
if request.method == 'POST':
rdata = {'status': False,
'message': '',
}
name = request.form['name'].strip()
if not name:
rdata['message'] = "Name is required"
rdata['status'] = 'error'
else:
# Check if organization name already exists
is_organization = Organization.query.filter_by(name=name).scalar()
if is_organization is not None:
rdata['message'] = "Organization with name {} already exists".format(name)
rdata['status'] = 'error'
else:
organization = Organization(name=name, user=current_user)
db.session.add(organization)
db.session.flush()
current_user.organizations.append(organization)
group = Group(name='Default', organization_id=organization.id)
db.session.add(group)
db.session.commit()
logger.info("User {} created organization {}".format(current_user.username,
organization.name))
data = [organization.serialize]
# Add current user to the new room
# TODO: Figure how to join a room here
# socketio.join_room('organization-' + str(organization.id))
socketio.emit('manage-organizations',
{'data': data, 'action': 'add'},
namespace='/manage/organizations',
room='organization-' + str(organization.id)
)
rdata['message'] = "Organization {} was successfully created".format(organization.name)
rdata['status'] = 'success'
return jsonify(rdata)
return render_template('manage/organizations.html')
def post(self):
rdata = {'success': False,
'message': ""
}
client_data = request.args
data = request.json
run = ScraperRun.query.filter_by(uuid=client_data['scraperRun']).scalar()
run.total_urls_hit = data.get('totalUrls')
run.ref_data_count = data.get('refDataCount')
run.ref_data_success_count = data.get('refDataSuccessCount')
run.num_rows_added_to_db = data.get('rowsAddedToDb')
run.stop_time = datetime.datetime.strptime(data['stopTime'], "%Y-%m-%d %H:%M:%S.%f")
# Calc runtime and get counts
runtime = run.stop_time - run.start_time
run.runtime = runtime.total_seconds()
counts = ScraperLog.query.filter_by(run_uuid=client_data['scraperRun'])
run.critical_count = counts.filter_by(level_name='CRITICAL').count()
run.error_count = counts.filter_by(level_name='ERROR').count()
run.warning_count = counts.filter_by(level_name='WARNING').count()
run.url_error_count = ScraperUrlError.query.filter_by(run_uuid=client_data['scraperRun']).count()
db.session.commit()
data = [run.serialize]
socketio.emit('data-scrapers',
{'data': data, 'action': 'stop'},
namespace='/data/scrapers/{env}/'.format(env=client_data['environment'].lower()),
room='organization-{org_id}'.format(org_id=run.scraper.group.organization_id)
)
rdata['success'] = True
rdata['message'] = ""
return rdata
def manage_organizations_delete(organization_id):
rdata = {'status': 'error',
'message': '',
}
organization = Organization.query.get(organization_id)
# Check if user can delete group
if current_user.id != organization.owner_id:
abort(403)
# Check if organization has scrapers still assigned to it
org_has_scrapers = False
for group in organization.groups:
if len(list(group.scrapers)) > 0:
org_has_scrapers = True
break
print(org_has_scrapers)
if organization.name == current_user.username:
rdata['message'] = "Cannot delete your private organization"
elif org_has_scrapers is True:
rdata['message'] = "Cannot delete. Organization has scrapers under it"
else:
db.session.delete(organization)
db.session.commit()
logger.info("User {username} deleted organization {org_name}"
.format(username=current_user.username, org_name=organization.name))
data = [organization.serialize]
socketio.emit('manage-organizations',
{'data': data, 'action': 'delete'},
namespace='/manage/organizations',
room='organization-{org_id}'.format(org_id=organization.id)
)
rdata['message'] = "Deleted organization {org_name}".format(org_name=organization.name)
rdata['status'] = 'success'
return jsonify(rdata)
def manage_groups():
if request.method == 'POST':
rdata = {'status': 'success',
'message': '',
}
name = request.form['name'].strip()
organization_id = request.form['organization'].strip()
if not name:
rdata['message'] = "Name is required"
rdata['status'] = 'error'
else:
# Check if group name for organization already exists
is_group = Group.query.filter_by(organization_id=organization_id).filter_by(name=name).scalar()
if is_group is not None:
rdata['message'] = "Group with name {} already exists".format(name)
rdata['status'] = 'error'
else:
# TODO: Make sure current_user has perms to create a group in this organization
group = Group(name=name, organization_id=organization_id)
group.organization = Organization.query.get(organization_id)
db.session.add(group)
db.session.commit()
logger.info("User {} created group {} in {}"
.format(current_user.email, group.name, group.organization.name))
data = [group.serialize]
socketio.emit('manage-groups',
{'data': data, 'action': 'add'},
namespace='/manage/groups',
room='organization-' + str(organization_id)
)
rdata['message'] = "Group {} was successfully created in {}".format(group.name,
group.organization.name)
rdata['status'] = 'success'
return jsonify(rdata)
return render_template('manage/groups.html')
def manage_scraper_edit():
rdata = {'success': False,
'message': "",
'displayAlert': None,
}
scraper_id = request.form['pk']
scraper_field = request.form['name'].strip()
scraper_new_value = request.form['value'].strip()
# Check if the user has permission to update this value
scraper = None
try:
scraper = Scraper.query.get(int(scraper_id))
scraper_users = scraper.group.organization.users
if current_user not in scraper_users or scraper is None:
rdata['success'] = False
rdata['message'] = "Invalid scraper"
return jsonify(rdata)
except Exception:
logger.exception("Error checking if user ({user_id}) can access the scraper ({scraper_id})"
.format(user_id=current_user.id, scraper_id=scraper_id))
rdata['success'] = False
rdata['message'] = "Invalid action"
return jsonify(rdata)
# Checks depending on the field being updated
if scraper_field == 'name':
# Scrapers can have the same name
# All error checking has been done, lets save the data
scraper.name = scraper_new_value
elif scraper_field == 'group':
try:
new_group = Group.query.get(int(scraper_new_value))
# Make sure user has permissions for the group
if current_user not in new_group.organization.users:
rdata['success'] = False
rdata['message'] = "Invalid group"
return jsonify(rdata)
# Check if the current owner is part of this new group, if not set the owner to current user
if scraper.owner not in new_group.organization.users:
rdata['displayAlert'] = {'status': 'warning',
'message': "User <b>{owner}</b> is not in in the selected "
"organization <b>{org_name}</b>"
"\nSetting scraper owner to <b>{current_user}</b>"
.format(owner=scraper.owner.username,
org_name=new_group.organization.name,
current_user=current_user.username,
),
}
# Default owner to current user
scraper.owner = current_user
# Set the new group
scraper.group = new_group
except Exception:
logger.exception("Error checking if user ({user_id}) can use the group ({new_val})"
.format(user_id=current_user.id, new_val=scraper_new_value))
rdata['success'] = False
rdata['message'] = "Invalid action"
return jsonify(rdata)
elif scraper_field == 'owner':
if scraper_new_value is None or scraper_new_value == '':
scraper.owner = None
else:
try:
new_user = User.query.get(int(scraper_new_value))
if new_user not in scraper_users or current_user not in scraper_users:
rdata['success'] = False
rdata['message'] = "Invalid user"
return jsonify(rdata)
# Everything looks good
scraper.owner = new_user
except Exception:
logger.exception("Error checking if user ({new_val}) is in the organization ({org_name})"
.format(new_val=scraper_new_value, org_name=scraper.group.organization.name))
rdata['success'] = False
rdata['message'] = "Invalid action"
return jsonify(rdata)
db.session.add(scraper)
db.session.commit()
rdata['success'] = True
logger.info("User {email} updated scraper {scraper_key} - {scraper_name}"
.format(email=current_user.email, scraper_key=scraper.key, scraper_name=scraper.name))
data = [scraper.serialize]
socketio.emit('manage-scrapers',
{'data': data, 'action': 'update'},
namespace='/manage/scrapers',
room='organization-{org_id}'.format(org_id=scraper.group.organization_id)
)
return jsonify(rdata)
def manage_scrapers():
if request.method == 'POST':
rdata = {'status': 'success',
'message': "",
}
name = request.form['name'].strip()
try:
group = Group.query.get(int(request.form['group'].strip()))
except ValueError:
group = None
try:
owner = User.query.get(int(request.form['owner'].strip()))
except ValueError:
owner = None
if not name:
rdata['message'] = "Name is required"
rdata['status'] = 'error'
if group is None:
rdata['message'] = "Group is required"
rdata['status'] = 'error'
elif owner is not None and owner not in group.organization.users:
# If the user is not in the org that the scraper is in
rdata['message'] = "Invalid owner for the scraper"
rdata['status'] = 'error'
else:
scraper = Scraper(name)
scraper.owner = owner
scraper.group = group
db.session.add(scraper)
# Flush to get the id so it can be encoded
db.session.flush()
# Do not change the string in the function below.
# It is just used to create the scraper key, nothing secure
scraper.key = generate_key(scraper.id, 'Scraper Salt 123')
db.session.add(scraper)
db.session.commit()
logger.info("User {} created scraper {} - {}"
.format(current_user.email, scraper.key, scraper.name))
data = [scraper.serialize]
socketio.emit('manage-scrapers',
{'data': data, 'action': 'add'},
namespace='/manage/scrapers',
room='organization-' + str(scraper.group.organization.id)
)
rdata['message'] = "Scraper {} was successfully created in group {}"\
.format(scraper.name, scraper.group.name)
rdata['status'] = 'success'
return jsonify(rdata)
group_list = []
for organization in current_user.organizations:
groups = Group.query.filter_by(organization=organization).all()
groups = [i.serialize for i in groups]
group_list.extend(groups)
return render_template('manage/scrapers.html',
groups=group_list,
)
def set(self, key, value):
try:
self.store[key] = value
socketio.emit(key, value, broadcast=True)
except KeyError:
socketio.send('error', 'KeyError')
