def user_create():
if not can_create_user(current_user):
abort(401)
form = UserForm(request.form)
# Need to add the Required flag back as it is globally removed during user_edit()
add_validator(form.password, Required)
fill_user_privileges(form.privilege.choices)
if request.method == 'POST' and form.validate():
db_session = DBSession()
user = get_user(db_session, form.username.data)
if user is not None:
return render_template('user/edit.html', form=form, duplicate_error=True)
user = User(
username=form.username.data,
password=form.password.data,
privilege=form.privilege.data,
fullname=form.fullname.data,
email=form.email.data)
user.preferences.append(Preferences())
db_session.add(user)
db_session.commit()
return redirect(url_for('home'))
else:
# Default to Active
form.active.data = True
return render_template('user/edit.html', form=form)
def refresh_all(cls):
"""
Retrieves all the catalog data and SMU XML file data and updates the database.
"""
db_session = DBSession()
catalog = SMUInfoLoader.get_catalog_from_cco()
if len(catalog) > 0:
system_option = SystemOption.get(db_session)
try:
# Remove all rows first
db_session.query(CCOCatalog).delete()
for platform in catalog:
releases = catalog[platform]
for release in releases:
cco_catalog = CCOCatalog(platform=platform,release=release)
db_session.add(cco_catalog)
SMUInfoLoader(platform, release)
system_option.cco_lookup_time = datetime.datetime.utcnow()
db_session.commit()
return True
except Exception:
logger.exception('refresh_all() hit exception')
db_session.rollback()
return False
def _collect_video_info(self,tid, simple_info, session:DBSession, collect_id = 0):
uploader_exists = session.query(
exists().where(database.UploaderInfo.mid == simple_info['author_id'])
).scalar()
if not uploader_exists:
new_up = database.UploaderInfo(
mid=simple_info['author_id'],
name=simple_info['author_name'],
)
session.add(new_up)
session.commit()
new_video = database.VideoInfo(
aid=simple_info['id'],
bvid=simple_info['bvid'],
tid=tid,
title=simple_info['name'],
owner_id=simple_info['author_id'],
collect_time=datetime.datetime.now(),
collect_id = collect_id
)
try:
session.add(new_video)
session.commit()
except Exception as e:
logger.error(f"向数据库中写入视频信息出错： {simple_info['name']}")
session.rollback()
def api_create_tar_job():
db_session = DBSession()
form = CreateTarForm(request.form)
server_id = request.args.get('server')
server_directory = request.args.get('server_directory')
source_tars = request.args.getlist('source_tars[]')
contents = request.args.getlist('tar_contents[]')
additional_packages = request.args.getlist('additional_packages[]')
new_tar_name = request.args.get('new_tar_name').strip('.tar')
create_tar_job = CreateTarJob(
server_id = server_id,
server_directory = server_directory,
source_tars = (',').join(source_tars),
contents = (',').join(contents),
additional_packages = (',').join(additional_packages),
new_tar_name = new_tar_name,
created_by = current_user.username,
status = 'Job Submitted.')
db_session.add(create_tar_job)
db_session.commit()
job_id = create_tar_job.id
return jsonify({'status': 'OK', 'job_id': job_id})
def command_profile_create():
db_session = DBSession()
form = CustomCommandProfileForm(request.form)
if request.method == 'POST' and form.validate():
command_profile = get_command_profile(db_session,
form.profile_name.data)
if command_profile is not None:
return render_template('custom_command/command_profile_edit.html',
form=form,
duplicate_error=True)
command_profile = CustomCommandProfile(
profile_name=form.profile_name.data,
command_list=','.join(
[l for l in form.command_list.data.splitlines() if l]),
created_by=current_user.username)
db_session.add(command_profile)
db_session.commit()
return redirect(url_for('custom_command.home'))
else:
return render_template('custom_command/command_profile_edit.html',
form=form)
def filesupload():
if request.method == 'POST':
file = request.files['file']
if file:
session = DBSession()
filename = datetime.datetime.now().strftime(
"%Y%m%d%H%M%S_") + secure_filename(file.filename)
user = current_user._get_current_object()
filepath = os.path.join(os.getcwd(), 'static', 'files',
'{}'.format(user.userid))
# now=time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(int(time.time())))
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(now)
print(filepath)
# file.save(os.path.join(os.getcwd(),'static',filename))
file.save(os.path.join(filepath, filename))
filerec = files(filename=filename,
filepath=filepath,
user_userid=user.userid,
uploadtime=now,
quote=1)
session.add(filerec)
session.commit()
record = filerecord(user_userid=user.userid,
files_fileid=filerec.fileid,
flag=1,
recordtime=now)
session.add(record)
session.commit()
session.close()
return 'File {0} has benn saved in {1} !'.format(
filename, filepath)
else:
return render_template('uploadtest.html')
def command_profile_create():
db_session = DBSession()
form = CustomCommandProfileForm(request.form)
if request.method == 'POST' and form.validate():
command_profile = get_command_profile(db_session, form.profile_name.data)
if command_profile is not None:
return render_template('custom_command/command_profile_edit.html',
form=form, duplicate_error=True)
command_profile = CustomCommandProfile(
profile_name=form.profile_name.data,
command_list=','.join([l for l in form.command_list.data.splitlines() if l]),
created_by=current_user.username
)
db_session.add(command_profile)
db_session.commit()
return redirect(url_for('custom_command.home'))
else:
return render_template('custom_command/command_profile_edit.html',
form=form)
def refresh_all(cls):
"""
Retrieves all the catalog data and SMU XML file data and updates the database.
"""
db_session = DBSession()
catalog = SMUInfoLoader.get_catalog_from_cco()
if len(catalog) > 0:
system_option = SystemOption.get(db_session)
try:
# Remove all rows first
db_session.query(CCOCatalog).delete()
for platform in catalog:
releases = catalog[platform]
for release in releases:
cco_catalog = CCOCatalog(platform=platform,
release=release)
db_session.add(cco_catalog)
# Now, retrieve from SMU XML file
SMUInfoLoader(platform, release, refresh=True)
system_option.cco_lookup_time = datetime.datetime.utcnow()
db_session.commit()
return True
except Exception:
logger.exception('refresh_all hit exception')
db_session.rollback()
return False
def software_profile_create():
# if not can_create_user(current_user):
# abort(401)
db_session = DBSession()
form = SoftwareProfileForm(request.form)
server_dialog_form = ServerDialogForm(request.form)
fill_servers(server_dialog_form.server_dialog_server.choices, get_server_list(db_session), False)
if request.method == 'POST' and form.validate():
software_profile = get_software_profile(db_session, form.profile_name.data)
if software_profile is not None:
return render_template('conformance/profile_edit.html',
form=form, system_option=SystemOption.get(db_session), duplicate_error=True)
software_profile = SoftwareProfile(
name=form.profile_name.data,
description=form.description.data,
packages=','.join([l for l in form.software_packages.data.splitlines() if l]),
created_by=current_user.username)
db_session.add(software_profile)
db_session.commit()
return redirect(url_for('conformance.home'))
else:
return render_template('conformance/profile_edit.html',
form=form, server_dialog_form=server_dialog_form,
system_option=SystemOption.get(db_session))
def insert_messages(messages):
from database import DBSession, Message, User
for index, message in enumerate(messages):
new_msg = Message(id=-index,
link='',
text='{}: {}'.format(message.sender,
message.message),
video='',
photo='',
audio='',
voice='',
type='text',
category='',
from_id=0,
date=message.date)
session = DBSession()
session.add(new_msg)
session.commit()
session.close()
deafult_user = User(id=0,
fullname='历史聊天记录',
username='******',
update_time=datetime.now())
session = DBSession()
session.add(deafult_user)
session.commit()
session.close()
def userregister():
if request.method == 'POST':
session = DBSession()
email = request.form['useremail']
username = request.form['username']
password = request.form['userpassword']
password = generate_password_hash(password)
print(email)
if session.query(User).filter(User.email == email).first():
return render_template('register.html', msg='该邮箱已被注册！')
else:
adduser = User(password=password, username=username, email=email, admin=0,activated=0)
session.add(adduser)
session.commit()
os.makedirs(os.path.join(os.getcwd(),'static','files','{0}'.format(adduser.userid)))
url='http://47.94.138.25/activate/{0}'.format(gen_token(adduser.userid))
# message=Message('Account authorize',sender='*****@*****.**',recipients=['{}'.format(username),'{}'.format(email)])
message = Message('Account authorize', sender='*****@*****.**',recipients=['*****@*****.**'])
message.body="Hello,{0}<br>This email is from admin!<br>Your account has not been activated yet!<br><a href='{1}'>click here to activate your account!</a><br>if you mistaken the email,please ignore it!".format(username,url)
mail.send(message)
session.close()
return redirect(url_for('users.userlogin'))
else:
return render_template('register.html')
def insert_messages(messages):
from database import DBSession, Message, User
for message in messages:
new_msg = Message(id=message.msgid,
link='https://t.me/c/{}/{}'.format(
LINK_ID, message.msgid),
text=message.message,
video='',
photo='',
audio='',
voice='',
type='text',
category='',
from_id=message.senderid,
date=message.date)
session = DBSession()
session.add(new_msg)
if not message.userexist:
new_user = User(id=message.senderid,
fullname=message.sender,
username='',
update_time=datetime.now())
session.add(new_user)
session.commit()
session.close()
def api_create_software_profile():
profile_name = request.form['profile_name']
description = request.form['description']
software_packages = request.form['software_packages']
db_session = DBSession()
software_profile = get_software_profile(db_session, profile_name)
if software_profile is not None:
return jsonify({
'status':
'Software profile "' + profile_name +
'" already exists. Use a different name instead.'
})
software_profile = SoftwareProfile(
name=profile_name,
description=description,
packages=','.join([l for l in software_packages.splitlines() if l]),
created_by=current_user.username)
db_session.add(software_profile)
db_session.commit()
return jsonify({'status': 'OK'})
def api_create_tar_job():
db_session = DBSession()
form = CreateTarForm(request.form)
server_id = request.args.get('server')
server_directory = request.args.get('server_directory')
source_tars = request.args.getlist('source_tars[]')
contents = request.args.getlist('tar_contents[]')
additional_packages = request.args.getlist('additional_packages[]')
new_tar_name = request.args.get('new_tar_name').replace('.tar', '')
create_tar_job = CreateTarJob(
server_id=server_id,
server_directory=server_directory,
source_tars=','.join(source_tars),
contents=','.join(contents),
additional_packages=','.join(additional_packages),
new_tar_name=new_tar_name,
created_by=current_user.username,
status='Job Submitted.')
db_session.add(create_tar_job)
db_session.commit()
job_id = create_tar_job.id
return jsonify({'status': 'OK', 'job_id': job_id})
def user_create():
if not can_create_user(current_user):
abort(401)
form = UserForm(request.form)
# Need to add the Required flag back as it is globally removed during user_edit()
add_validator(form.password, Required)
fill_user_privileges(form.privilege.choices)
if request.method == 'POST' and form.validate():
db_session = DBSession()
user = get_user(db_session, form.username.data)
if user is not None:
return render_template('user/edit.html',
form=form,
duplicate_error=True)
user = User(username=form.username.data,
password=form.password.data,
privilege=form.privilege.data,
fullname=form.fullname.data,
email=form.email.data)
user.preferences.append(Preferences())
db_session.add(user)
db_session.commit()
return redirect(url_for('home'))
else:
# Default to Active
form.active.data = True
return render_template('user/edit.html', form=form)
def get_smu_info_from_cco(self, platform, release):
save_to_db = True
db_session = DBSession()
platform_release = platform + '_' + release
try:
self.smu_meta = SMUMeta(platform_release=platform_release)
# Load data from the SMU XML file
self.load()
# This can happen if the given platform and release is not valid.
# The load() method calls get_smu_info_from_db and failed.
if not self.is_valid:
logger.error('get_smu_info_from_cco() hit exception, platform_release=' + platform_release)
return
db_smu_meta = db_session.query(SMUMeta).filter(SMUMeta.platform_release == platform_release).first()
if db_smu_meta:
if db_smu_meta.created_time == self.smu_meta.created_time:
save_to_db = False
else:
# Delete the existing smu_meta and smu_info for this platform and release
db_session.delete(db_smu_meta)
db_session.commit()
if save_to_db:
db_session.add(self.smu_meta)
else:
db_smu_meta.retrieval_time = datetime.datetime.utcnow()
db_session.commit()
except Exception:
logger.exception('get_smu_info_from_cco() hit exception, platform_release=' + platform_release)
def init():
session = DBSession()
if session.query(Config).filter().count() == 0:
user = USER + ':' + PASSWORD
movie_dir_re = os.environ['MOVIE_DIR_RE']
tg_push_on = True if os.environ['TG_ON'] == 'true' else False
tg_chatid = os.environ['TG_CHATID']
tg_bot_token = os.environ['TG_BOT_TOKEN']
bark_push_on = True if os.environ['BARK_ON'] == 'true' else False
bark_tokens = os.environ['BARK_TOKENS']
server_cyann_on = True if os.environ[
'SERVER_CYANN'] == 'true' else False
server_cyann_token = os.environ['SERVER_CYANN_TOKEN']
proxy_on = False
proxy_url = ''
config = Config(user=user,
root_dir='/mnt/media',
movie_dir_re=movie_dir_re,
tg_push_on=tg_push_on,
tg_chatid=tg_chatid,
tg_bot_token=tg_bot_token,
bark_push_on=bark_push_on,
bark_tokens=bark_tokens,
server_cyann_on=server_cyann_on,
server_cyann_token=server_cyann_token,
proxy_on=proxy_on,
proxy_url=proxy_url)
session.add(config)
session.commit()
session.close()
def insert_user_or_do_nothing(user_id, fullname, username):
session = DBSession()
target_user = session.query(User).get(user_id)
if not target_user:
new_user = User(id=user_id, fullname=fullname, username=username)
session.add(new_user)
session.commit()
session.close()
def insert_chat_or_do_nothing(chat_id, title):
session = DBSession()
target_chat = session.query(Chat).get(chat_id)
if not target_chat:
new_chat = Chat(id=chat_id, title=title, enable=False)
session.add(new_chat)
session.commit()
session.close()
def init_encrypt():
global encrypt_dict
db_session = DBSession()
if db_session.query(Encrypt).count() == 0:
db_session.add(Encrypt())
db_session.commit()
encrypt_dict = dict(Encrypt.get(db_session).__dict__)
def init_encrypt():
global encrypt_dict
db_session = DBSession()
if db_session.query(Encrypt).count() == 0:
db_session.add(Encrypt())
db_session.commit()
encrypt_dict = dict(Encrypt.get(db_session).__dict__)
def init_sys_time():
db_session = DBSession()
if db_session.query(System).count() == 0:
db_session.add(System())
db_session.commit()
else:
system = db_session.query(System).first()
system.start_time = datetime.datetime.utcnow()
db_session.commit()
def init_sys_time():
db_session = DBSession()
if db_session.query(System).count() == 0:
db_session.add(System())
db_session.commit()
else:
system = db_session.query(System).first()
system.start_time = datetime.datetime.utcnow()
db_session.commit()
def insert_or_update_user(user_id, fullname, username):
session = DBSession()
target_user = session.query(User).get(user_id)
if not target_user:
new_user = User(id=user_id, fullname=fullname, username=username)
session.add(new_user)
session.commit()
elif target_user.fullname != fullname or target_user.username != username:
target_user.fullname = fullname
target_user.username = username
session.commit()
session.close()
def disbale_chat_or_do_nothing(chat_id):
session = DBSession()
target_chat = session.query(Chat).get(chat_id)
if target_chat and target_chat.enable:
target_chat.enable = False
session.add(target_chat)
session.commit()
msg_text = '成功停用,记录暂时保留,删除需使用/delete命令'
else:
msg_text = '此前已停用/未启用,启用需使用/start命令'
session.close()
return msg_text
def insert_messages(chat_id, messages):
fail_count = 0
fail_messages = []
success_count = 0
for message in messages:
if 'from_id' not in message:
continue
elif 'user' not in message['from_id']:
continue
insert_user_or_do_nothing(message['from_id'][4:], message['from'],
message['from'])
if isinstance(message['text'], list):
msg_text = ''
for obj in message['text']:
if isinstance(obj, dict):
msg_text += obj['text']
else:
msg_text += obj
else:
msg_text = message['text']
if msg_text == '':
msg_text == '[其他消息]'
message_date = datetime.strptime(message['date'], '%Y-%m-%dT%H:%M:%S')
new_msg = Message(id=message['id'],
link='https://t.me/c/{}/{}'.format(
chat_id, message['id']),
text=msg_text,
video='',
photo='',
audio='',
voice='',
type='text',
category='',
from_id=message['from_id'][4:],
from_chat=chat_id,
date=message_date)
session = DBSession()
try:
session.add(new_msg)
session.commit()
success_count += 1
except Exception as e:
print(e)
fail_count += 1
fail_messages.append(str(message))
session.close()
return success_count, fail_count, fail_messages
def insert_message(msg_id, msg_text, msg_video, msg_photo, msg_audio,
msg_voice, msg_type, from_id, date):
new_msg = Message(id=msg_id,
text=msg_text,
video=msg_video,
photo=msg_photo,
audio=msg_audio,
voice=msg_voice,
type=msg_type,
category='',
from_id=from_id,
date=date)
session = DBSession()
session.add(new_msg)
session.commit()
session.close()
def useraccfriend(userid):
if request.method == 'GET':
session = DBSession()
user = current_user._get_current_object()
if userid == user.userid:
return Response(response='Failed!', status=400)
else:
now=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
userre = userrelationship(user=user.userid, friend=userid, flag=1,rtime=now)
session.add(userre)
fr=session.query(userrelationship).filter(userrelationship.user == userid,
userrelationship.friend == user.userid).first()
fr.flag=1
fr.rtime=now
session.commit()
session.close()
return Response(response='successfully!', status=200)
def insert_chat_or_enable(chat_id, title):
session = DBSession()
target_chat = session.query(Chat).get(chat_id)
if not target_chat:
new_chat = Chat(id=chat_id, title=title, enable=True)
session.add(new_chat)
session.commit()
msg_text = '成功启用'
else:
if target_chat.enable:
msg_text = '前期已启用,停用需使用/stop命令'
else:
target_chat.enable = True
session.commit()
msg_text = '成功恢复使用'
session.close()
return msg_text
def convert_config_file():
filename = request.args.get('filename', '', type=str)
filename = secure_filename(filename)
config_conversion_path = get_config_conversion_path()
db_session = DBSession()
convert_config_job = ConvertConfigJob(file_path=os.path.join(config_conversion_path, filename),
status='Preparing the conversion')
db_session.add(convert_config_job)
db_session.commit()
job_id = convert_config_job.id
return jsonify({'status': 'OK', 'job_id': job_id})
def convert_config_file():
filename = request.args.get('filename', '', type=str)
filename = secure_filename(filename)
config_conversion_path = get_config_conversion_path()
db_session = DBSession()
convert_config_job = ConvertConfigJob(file_path=os.path.join(
config_conversion_path, filename),
status='Preparing the conversion')
db_session.add(convert_config_job)
db_session.commit()
job_id = convert_config_job.id
return jsonify({'status': 'OK', 'job_id': job_id})
def emit(self, record):
trace = traceback.format_exc() if record.__dict__['exc_info'] else None
args = record.__dict__['args']
msg = record.__dict__['msg']
if len(args) >= 1:
msg = msg % args
log = Log(level=record.__dict__['levelname'],
trace=trace,
msg=msg,
created_time=datetime.datetime.utcnow())
db_session = DBSession()
db_session.add(log)
db_session.commit()
def emit(self, record):
trace = traceback.format_exc() if record.__dict__['exc_info'] else None
args = record.__dict__['args']
msg = record.__dict__['msg']
if len(args) >= 1:
msg = msg % args
log = Log(
level=record.__dict__['levelname'],
trace=trace,
msg=msg,
created_time=datetime.datetime.utcnow())
db_session = DBSession()
db_session.add(log)
db_session.commit()
def write_to_database(self):
""" 将CheckUpdate实例的info_dic数据写入数据库 """
session = DBSession()
try:
if self.name in {x.ID for x in session.query(Saved).all()}:
saved_data = session.query(Saved).filter(Saved.ID == self.name).one()
saved_data.FULL_NAME = self.fullname
for key, value in self.__info_dic.items():
setattr(saved_data, key, value)
else:
new_data = Saved(
ID=self.name,
FULL_NAME=self.fullname,
**self.__info_dic
)
session.add(new_data)
session.commit()
finally:
session.close()
def useraddfriend(email):
if request.method == 'GET':
session = DBSession()
user = current_user._get_current_object()
friend=session.query(User).filter(User.email == email).first()
userid=friend.userid
if session.query(userrelationship).filter(userrelationship.user == user.userid,
userrelationship.friend == userid).first():
# user has been send request
return Response(response='You have been send request or already friend!', status=400)
if userid == user.userid:
return Response(response='Failed!', status=400)
else:
# now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
userre = userrelationship(user=user.userid, friend=userid, flag=0,rtime=now)
session.add(userre)
session.commit()
session.close()
return Response(response='successfully!', status=200)
def get_database_file(bot, update):
if update.message.chat_id != config.GROUP_ID:
return
delete_prev_message(bot, update)
now = datetime.datetime.now()
session = DBSession()
count = session.query(DBFile).filter().count()
if count:
last_db_file = session.query(DBFile).filter().order_by(
DBFile.date.desc()).first()
else:
last_db_file = None
if last_db_file and last_db_file.date + datetime.timedelta(
minutes=10) > now:
file = last_db_file.file_id
filename = 'bot.{}.db'.format(
last_db_file.date.strftime("%Y-%m-%d_%H.%M"))
new_record = False
else:
file = open('bot.db', 'rb')
filename = 'bot.{}.db'.format(now.strftime("%Y-%m-%d_%H.%M"))
new_record = True
sent_message = bot.send_document(chat_id=update.message.chat_id,
document=file,
filename=filename,
disable_notification=True)
# 保存新的获取数据库记录
if new_record:
file_id = sent_message.document.file_id
new_db_file = DBFile(file_id=file_id, date=now)
session.add(new_db_file)
session.commit()
session.close()
return sent_message
def software_profile_create():
# if not can_create_user(current_user):
# abort(401)
db_session = DBSession()
form = SoftwareProfileForm(request.form)
server_dialog_form = ServerDialogForm(request.form)
fill_servers(server_dialog_form.server_dialog_server.choices,
get_server_list(db_session), False)
if request.method == 'POST' and form.validate():
software_profile = get_software_profile(db_session,
form.profile_name.data)
if software_profile is not None:
return render_template('conformance/profile_edit.html',
form=form,
system_option=SystemOption.get(db_session),
duplicate_error=True)
software_profile = SoftwareProfile(
name=form.profile_name.data,
description=form.description.data,
packages=','.join(
[l for l in form.software_packages.data.splitlines() if l]),
created_by=current_user.username)
db_session.add(software_profile)
db_session.commit()
return redirect(url_for('conformance.home'))
else:
return render_template('conformance/profile_edit.html',
form=form,
server_dialog_form=server_dialog_form,
system_option=SystemOption.get(db_session))
def get_smu_info_from_cco(self, platform, release):
same_as_db = False
db_session = DBSession()
platform_release = platform + '_' + release
try:
self.smu_meta = SMUMeta(platform_release=platform_release)
# Load data from the SMU XML file
self.load()
# This can happen if the given platform and release is not valid.
# The load method calls get_smu_info_from_db and failed.
if self.smu_meta is None:
return
db_smu_meta = db_session.query(SMUMeta).filter(
SMUMeta.platform_release == platform + '_' + release).first()
if db_smu_meta is not None:
if db_smu_meta.created_time == self.smu_meta.created_time:
same_as_db = True
else:
# Delete the existing smu_meta and smu_info for this platform and release
db_session.delete(db_smu_meta)
if not same_as_db:
db_session.add(self.smu_meta)
else:
db_smu_meta.retrieval_time = datetime.datetime.utcnow()
# Use Flush to detect concurrent saving condition. It is
# possible that another process may perform the same save.
# If this happens, Duplicate Key may result.
db_session.flush()
db_session.commit()
except IntegrityError:
db_session.rollback()
except Exception:
db_session.rollback()
logger.exception('get_smu_info_from_cco hit exception')
def sendfile(recuserid, fileid, note):
session = DBSession()
senduser = current_user._get_current_object()
print(senduser)
print(recuserid, fileid, note)
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if note is None or note == 'None': note = '00'
if session.query(User).filter(User.userid == recuserid).first(
) and session.query(filerecord).filter(
filerecord.files_fileid == fileid, filerecord.flag == 1,
filerecord.user_userid
== senduser.userid).first() and recuserid != senduser.userid and (
session.query(userrelationship).filter(
userrelationship.user == recuserid, userrelationship.friend
== senduser.userid).first()
and session.query(userrelationship).filter(
userrelationship.user == senduser.userid,
userrelationship.friend == recuserid).first()):
src = sendrecord(user_userid=senduser.userid,
user_userid1=recuserid,
files_fileid=fileid,
sendtime=now,
note=note)
session.add(src)
session.commit()
recfile = filerecord(user_userid=recuserid,
files_fileid=fileid,
flag=1,
recordtime=now)
session.add(recfile)
session.commit()
filequote = session.query(files).filter(files.fileid == fileid).first()
filequote.quote += 1
filename = filequote.filename
session.commit()
session.close()
return 'send file:{0} successfully!'.format(filename)
else:
return 'send fail!'
def get_smu_info_from_cco(self, platform, release):
save_to_db = True
db_session = DBSession()
platform_release = platform + '_' + release
try:
self.smu_meta = SMUMeta(platform_release=platform_release)
# Load data from the SMU XML file
self.load()
# This can happen if the given platform and release is not valid.
# The load() method calls get_smu_info_from_db and failed.
if not self.is_valid:
logger.error(
'get_smu_info_from_cco() hit exception, platform_release='
+ platform_release)
return
db_smu_meta = db_session.query(SMUMeta).filter(
SMUMeta.platform_release == platform_release).first()
if db_smu_meta:
if db_smu_meta.created_time == self.smu_meta.created_time:
save_to_db = False
else:
# Delete the existing smu_meta and smu_info for this platform and release
db_session.delete(db_smu_meta)
db_session.commit()
if save_to_db:
db_session.add(self.smu_meta)
else:
db_smu_meta.retrieval_time = datetime.datetime.utcnow()
db_session.commit()
except Exception:
logger.exception(
'get_smu_info_from_cco() hit exception, platform_release=' +
platform_release)
def get_smu_info_from_cco(self, platform, release):
same_as_db = False
db_session = DBSession()
platform_release = platform + '_' + release
try:
self.smu_meta = SMUMeta(platform_release=platform_release)
# Load data from the SMU XML file
self.load()
# This can happen if the given platform and release is not valid.
# The load method calls get_smu_info_from_db and failed.
if self.smu_meta is None:
return
db_smu_meta = db_session.query(SMUMeta).filter(SMUMeta.platform_release == platform + '_' + release).first()
if db_smu_meta is not None:
if db_smu_meta.created_time == self.smu_meta.created_time:
same_as_db = True
else:
# Delete the existing smu_meta and smu_info for this platform and release
db_session.delete(db_smu_meta)
if not same_as_db:
db_session.add(self.smu_meta)
else:
db_smu_meta.retrieval_time = datetime.datetime.utcnow()
# Use Flush to detect concurrent saving condition. It is
# possible that another process may perform the same save.
# If this happens, Duplicate Key may result.
db_session.flush()
db_session.commit()
except IntegrityError:
db_session.rollback()
except Exception:
db_session.rollback()
logger.exception('get_smu_info_from_cco() hit exception')
def api_create_software_profile():
profile_name = request.form['profile_name']
description = request.form['description']
software_packages = request.form['software_packages']
db_session = DBSession()
software_profile = get_software_profile(db_session, profile_name)
if software_profile is not None:
return jsonify({'status': 'Software profile "' + profile_name +
'" already exists. Use a different name instead.'})
software_profile = SoftwareProfile(
name=profile_name,
description=description,
packages=','.join([l for l in software_packages.splitlines() if l]),
created_by=current_user.username)
db_session.add(software_profile)
db_session.commit()
return jsonify({'status': 'OK'})
def home():
if current_user.privilege != UserPrivilege.ADMIN:
abort(401)
db_session = DBSession()
smtp_form = SMTPForm(request.form)
admin_console_form = AdminConsoleForm(request.form)
smtp_server = get_smtp_server(db_session)
system_option = SystemOption.get(db_session)
fill_user_privileges(admin_console_form.ldap_default_user_privilege.choices)
if request.method == 'POST' and \
smtp_form.validate() and \
admin_console_form.validate():
if smtp_server is None:
smtp_server = SMTPServer()
db_session.add(smtp_server)
smtp_server.server = smtp_form.server.data
smtp_server.server_port = smtp_form.server_port.data if len(smtp_form.server_port.data) > 0 else None
smtp_server.sender = smtp_form.sender.data
smtp_server.use_authentication = smtp_form.use_authentication.data
smtp_server.username = smtp_form.username.data
if len(smtp_form.password.data) > 0:
smtp_server.password = smtp_form.password.data
smtp_server.secure_connection = smtp_form.secure_connection.data
system_option.inventory_threads = admin_console_form.num_inventory_threads.data
system_option.install_threads = admin_console_form.num_install_threads.data
system_option.download_threads = admin_console_form.num_download_threads.data
system_option.can_schedule = admin_console_form.can_schedule.data
system_option.can_install = admin_console_form.can_install.data
system_option.enable_email_notify = admin_console_form.enable_email_notify.data
system_option.enable_inventory = admin_console_form.enable_inventory.data
# The LDAP UI may be hidden if it is not supported.
# In this case, the flag is not set.
if not is_empty(admin_console_form.enable_ldap_auth.data):
system_option.enable_ldap_auth = admin_console_form.enable_ldap_auth.data
system_option.ldap_server_url = admin_console_form.ldap_server_url.data
system_option.ldap_default_user_privilege = admin_console_form.ldap_default_user_privilege.data
system_option.ldap_server_distinguished_names = admin_console_form.ldap_server_distinguished_names.data.strip()
system_option.inventory_hour = admin_console_form.inventory_hour.data
system_option.inventory_history_per_host = admin_console_form.inventory_history_per_host.data
system_option.download_history_per_user = admin_console_form.download_history_per_user.data
system_option.install_history_per_host = admin_console_form.install_history_per_host.data
system_option.total_system_logs = admin_console_form.total_system_logs.data
system_option.enable_default_host_authentication = admin_console_form.enable_default_host_authentication.data
system_option.default_host_authentication_choice = admin_console_form.default_host_authentication_choice.data
system_option.enable_cco_lookup = admin_console_form.enable_cco_lookup.data
system_option.use_utc_timezone = admin_console_form.use_utc_timezone.data
system_option.default_host_username = admin_console_form.default_host_username.data
if len(admin_console_form.default_host_password.data) > 0:
system_option.default_host_password = admin_console_form.default_host_password.data
system_option.enable_user_credential_for_host = admin_console_form.enable_user_credential_for_host.data
db_session.commit()
return redirect(url_for('home'))
else:
admin_console_form.num_inventory_threads.data = system_option.inventory_threads
admin_console_form.num_install_threads.data = system_option.install_threads
admin_console_form.num_download_threads.data = system_option.download_threads
admin_console_form.can_schedule.data = system_option.can_schedule
admin_console_form.can_install.data = system_option.can_install
admin_console_form.enable_email_notify.data = system_option.enable_email_notify
admin_console_form.enable_ldap_auth.data = system_option.enable_ldap_auth
admin_console_form.ldap_server_url.data = system_option.ldap_server_url
admin_console_form.ldap_default_user_privilege.data = system_option.ldap_default_user_privilege
admin_console_form.ldap_server_distinguished_names.data = system_option.ldap_server_distinguished_names
admin_console_form.enable_inventory.data = system_option.enable_inventory
admin_console_form.inventory_hour.data = system_option.inventory_hour
admin_console_form.inventory_history_per_host.data = system_option.inventory_history_per_host
admin_console_form.download_history_per_user.data = system_option.download_history_per_user
admin_console_form.install_history_per_host.data = system_option.install_history_per_host
admin_console_form.total_system_logs.data = system_option.total_system_logs
admin_console_form.enable_default_host_authentication.data = system_option.enable_default_host_authentication
admin_console_form.default_host_authentication_choice.data = system_option.default_host_authentication_choice
admin_console_form.default_host_username.data = system_option.default_host_username
admin_console_form.enable_cco_lookup.data = system_option.enable_cco_lookup
admin_console_form.use_utc_timezone.data = system_option.use_utc_timezone
admin_console_form.cco_lookup_time.data = get_datetime_string(system_option.cco_lookup_time)
admin_console_form.enable_user_credential_for_host.data = system_option.enable_user_credential_for_host
if not is_empty(system_option.default_host_password):
admin_console_form.default_host_password_placeholder = 'Use Password on File'
else:
admin_console_form.default_host_password_placeholder = 'No Password Specified'
if smtp_server is not None:
smtp_form.server.data = smtp_server.server
smtp_form.server_port.data = smtp_server.server_port
smtp_form.sender.data = smtp_server.sender
smtp_form.use_authentication.data = smtp_server.use_authentication
smtp_form.username.data = smtp_server.username
smtp_form.secure_connection.data = smtp_server.secure_connection
if not is_empty(smtp_server.password):
smtp_form.password_placeholder = 'Use Password on File'
else:
smtp_form.password_placeholder = 'No Password Specified'
return render_template('admin/index.html',
admin_console_form=admin_console_form,
smtp_form=smtp_form,
system_option=SystemOption.get(db_session),
is_ldap_supported=is_ldap_supported())
def api_import_hosts():
region_id = int(request.form['region_id'])
jump_host_id = int(request.form['jump_host_id'])
software_profile_id = int(request.form['software_profile_id'])
data_list = request.form['data_list']
db_session = DBSession()
if region_id == -1:
return jsonify({'status': 'Region has not been specified.'})
if region_id > 0:
region = get_region_by_id(db_session, region_id)
if region is None:
return jsonify({'status': 'Region is no longer exists in the database.'})
if jump_host_id > 0:
jump_host = get_jump_host_by_id(db_session, jump_host_id)
if jump_host is None:
return jsonify({'status': 'Jump Host is no longer exists in the database.'})
if software_profile_id > 0:
software_profile = get_software_profile_by_id(db_session, software_profile_id)
if software_profile is None:
return jsonify({'status': 'Software Profile is no longer exists in the database.'})
error = []
reader = csv.reader(data_list.splitlines(), delimiter=',')
header_row = next(reader)
# header_row: ['hostname', 'location', 'roles', 'ip', 'username', 'password', 'connection', 'port']
# Check mandatory data fields
if HEADER_FIELD_HOSTNAME not in header_row:
error.append('"hostname" is missing in the header.')
if HEADER_FIELD_IP not in header_row:
error.append('"ip" is missing in the header.')
if HEADER_FIELD_CONNECTION not in header_row:
error.append('"connection" is missing in the header.')
for header_field in header_row:
if header_field not in HEADER_FIELDS:
error.append('"' + header_field + '" is not a correct header field.')
if error:
return jsonify({'status': '\n'.join(error)})
error = []
data_list = list(reader)
region_dict = get_region_name_to_id_dict(db_session)
# Check if each row has the same number of data fields as the header
row = 2
for row_data in data_list:
if len(row_data) != len(header_row):
error.append('line {} has wrong number of data fields - {}.'.format(row, row_data))
else:
hostname = get_acceptable_string(get_row_data(row_data, header_row, HEADER_FIELD_HOSTNAME))
if is_empty(hostname):
error.append('line {} has invalid hostname - {}.'.format(row, row_data))
# Validate the connection type
connection_type = get_row_data(row_data, header_row, HEADER_FIELD_CONNECTION)
if is_empty(connection_type) or connection_type not in [ConnectionType.TELNET, ConnectionType.SSH]:
error.append('line {} has a wrong connection type (should either be "telnet" or "ssh") - {}.'.format(row, row_data))
region_name = get_acceptable_string(get_row_data(row_data, header_row, HEADER_FIELD_REGION))
if region_name is not None:
# No blank region is allowed
if len(region_name) == 0:
error.append('line {} has no region specified - {}.'.format(row, row_data))
else:
if region_name not in region_dict.keys():
# Create the new region
try:
region = Region(name=region_name, created_by=current_user.username)
db_session.add(region)
db_session.commit()
# Add to region dictionary for caching purpose.
region_dict[region_name] = region.id
except Exception as e:
logger.exception('api_import_hosts() hit exception')
error.append('Unable to create region {} - {}.'.format(region_name, e.message))
row += 1
if error:
return jsonify({'status': '\n'.join(error)})
# Import the data
row = 2
for row_data in data_list:
try:
created_by = current_user.username
hostname = get_acceptable_string(get_row_data(row_data, header_row, HEADER_FIELD_HOSTNAME))
# Check if the host already exists in the database.
host = get_host(db_session, hostname)
region_name = get_acceptable_string(get_row_data(row_data, header_row, HEADER_FIELD_REGION))
if region_name is None:
alternate_region_id = region_id
else:
alternate_region_id = region_dict[region_name]
location = get_row_data(row_data, header_row, HEADER_FIELD_LOCATION)
if host and location is None:
location = host.location
roles = get_row_data(row_data, header_row, HEADER_FIELD_ROLES)
if host and roles is None:
roles = host.roles
host_or_ip = get_row_data(row_data, header_row, HEADER_FIELD_IP)
if host and host_or_ip is None:
host_or_ip = host.connection_param[0].host_or_ip
connection_type = get_row_data(row_data, header_row, HEADER_FIELD_CONNECTION)
if host and connection_type is None:
connection_type = host.connection_param[0].connection_type
username = get_row_data(row_data, header_row, HEADER_FIELD_USERNAME)
if host and username is None:
username = host.connection_param[0].username
password = get_row_data(row_data, header_row, HEADER_FIELD_PASSWORD)
if host and password is None:
password = host.connection_param[0].password
enable_password = get_row_data(row_data, header_row, HEADER_FIELD_ENABLE_PASSWORD)
if host and enable_password is None:
enable_password = host.connection_param[0].enable_password
port_number = get_row_data(row_data, header_row, HEADER_FIELD_PORT)
if host and port_number is None:
port_number = host.connection_param[0].port_number
# If no software profile is selected, retain the existing one instead of overwriting it.
if host and (software_profile_id is None or software_profile_id <= 0):
alternate_software_profile_id = host.software_profile_id
else:
alternate_software_profile_id = software_profile_id
# If no jump host is selected, retain the existing one instead of overwriting it.
if host and (jump_host_id is None or jump_host_id <= 0):
alternate_jump_host_id = host.connection_param[0].jump_host_id
else:
alternate_jump_host_id = jump_host_id
create_or_update_host(db_session, hostname, alternate_region_id, location, roles,
alternate_software_profile_id, connection_type, host_or_ip,
username, password, enable_password, port_number,
alternate_jump_host_id, created_by, host)
except Exception as e:
return jsonify({'status': 'Line {} - {} - {}.'.format(row, e.message, row_data)})
row += 1
return jsonify({'status': 'OK'})
def run_conformance_report(profile_name, match_criteria, hostnames):
host_not_in_conformance = 0
host_out_dated_inventory = 0
db_session = DBSession()
software_profile = get_software_profile(db_session, profile_name)
if software_profile is not None:
software_profile_packages = software_profile.packages.split(',')
conformance_report = ConformanceReport(
software_profile=software_profile.name,
software_profile_packages=','.join(sorted(software_profile_packages)),
match_criteria=match_criteria,
hostnames=hostnames,
user_id=current_user.id,
created_by=current_user.username)
for hostname in hostnames.split(','):
host = get_host(db_session, hostname)
if host:
packages_to_match = [software_profile_package.replace('.pie', '')
for software_profile_package in software_profile_packages]
inventory_job = host.inventory_job[0]
comments = ''
if inventory_job is not None and inventory_job.last_successful_time is not None:
comments = '(' + get_last_successful_inventory_elapsed_time(host) + ')'
if inventory_job.status != JobStatus.COMPLETED:
comments += ' *'
host_out_dated_inventory += 1
host_packages = []
if match_criteria == 'inactive':
host_packages = get_host_inactive_packages(hostname)
elif match_criteria == 'active':
host_packages = get_host_active_packages(hostname)
missing_packages = get_missing_packages(host_packages, software_profile_packages)
if missing_packages:
host_not_in_conformance += 1
conformed = False
if len(host_packages) > 0 and len(missing_packages) == 0:
conformed = True
conformance_report_entry = ConformanceReportEntry(
hostname=hostname,
platform='Unknown' if host.software_platform is None else host.software_platform,
software='Unknown' if host.software_version is None else host.software_version,
conformed='Yes' if conformed else 'No',
comments=comments,
host_packages=','.join(sorted(match_packages(host_packages, packages_to_match))),
missing_packages=','.join(sorted(missing_packages)))
else:
# Flag host not found condition
host_out_dated_inventory += 1
host_not_in_conformance += 1
conformance_report_entry = ConformanceReportEntry(
hostname=hostname,
platform='MISSING',
software='MISSING',
inventory_status='MISSING',
last_successful_retrieval='MISSING',
host_packages='MISSING',
missing_packages='MISSING')
conformance_report.entries.append(conformance_report_entry)
conformance_report.host_not_in_conformance = host_not_in_conformance
conformance_report.host_out_dated_inventory = host_out_dated_inventory
db_session.add(conformance_report)
db_session.commit()
else:
return jsonify({'status': 'Unable to locate the software profile %s' % profile_name})
purge_old_conformance_reports(db_session)
return jsonify({'status': 'OK'})
def api_import_hosts():
importable_header = [HEADER_FIELD_HOSTNAME, HEADER_FIELD_REGION, HEADER_FIELD_ROLES, HEADER_FIELD_IP,
HEADER_FIELD_USERNAME, HEADER_FIELD_PASSWORD, HEADER_FIELD_CONNECTION, HEADER_FIELD_PORT]
region_id = request.form['region']
data_list = request.form['data_list']
db_session = DBSession()
selected_region = get_region_by_id(db_session, region_id)
if selected_region is None:
return jsonify({'status': 'Region is no longer exists in the database.'})
# Check mandatory data fields
error = []
reader = csv.reader(data_list.splitlines(), delimiter=',')
header_row = next(reader)
if HEADER_FIELD_HOSTNAME not in header_row:
error.append('"hostname" is missing in the header.')
if HEADER_FIELD_IP not in header_row:
error.append('"ip" is missing in the header.')
if HEADER_FIELD_CONNECTION not in header_row:
error.append('"connection" is missing in the header.')
for header_field in header_row:
if header_field not in importable_header:
error.append('"' + header_field + '" is not a correct header field.')
if error:
return jsonify({'status': ','.join(error)})
# Check if each row has the same number of data fields as the header
error = []
data_list = list(reader)
row = 2
COLUMN_CONNECTION = get_column_number(header_row, HEADER_FIELD_CONNECTION)
COLUMN_REGION = get_column_number(header_row, HEADER_FIELD_REGION)
for row_data in data_list:
if len(row_data) > 0:
if len(row_data) != len(header_row):
error.append('line %d has wrong number of data fields.' % row)
else:
if COLUMN_CONNECTION >= 0:
# Validate the connection type
data_field = row_data[COLUMN_CONNECTION]
if data_field != ConnectionType.TELNET and data_field != ConnectionType.SSH:
error.append('line %d has a wrong connection type (should either be "telnet" or "ssh").' % row)
if COLUMN_REGION >= 0:
# Create a region if necessary
data_field = get_acceptable_string(row_data[COLUMN_REGION])
region = get_region(db_session, data_field)
if region is None and data_field:
try:
db_session.add(Region(name=data_field,
created_by=current_user.username))
db_session.commit()
except Exception:
db_session.rollback()
error.append('Unable to create region %s.' % data_field)
row += 1
if error:
return jsonify({'status': ','.join(error)})
# Import the data
error = []
im_regions = {}
for data in data_list:
if len(data) == 0:
continue
db_host = None
im_host = Host()
im_host.region_id = selected_region.id
im_host.created_by = current_user.username
im_host.inventory_job.append(InventoryJob())
im_host.context.append(HostContext())
im_host.connection_param.append(ConnectionParam())
im_host.connection_param[0].username = ''
im_host.connection_param[0].password = ''
im_host.connection_param[0].port_number = ''
for column in range(len(header_row)):
header_field = header_row[column]
data_field = data[column].strip()
if header_field == HEADER_FIELD_HOSTNAME:
hostname = get_acceptable_string(data_field)
db_host = get_host(db_session, hostname)
im_host.hostname = hostname
elif header_field == HEADER_FIELD_REGION:
region_name = get_acceptable_string(data_field)
if region_name in im_regions:
im_host.region_id = im_regions[region_name]
else:
region = get_region(db_session, region_name)
if region is not None:
im_host.region_id = region.id
# Saved for later lookup
im_regions[region_name] = region.id
elif header_field == HEADER_FIELD_ROLES:
im_host.roles = remove_extra_spaces(data_field)
elif header_field == HEADER_FIELD_IP:
im_host.connection_param[0].host_or_ip = remove_extra_spaces(data_field)
elif header_field == HEADER_FIELD_USERNAME:
username = get_acceptable_string(data_field)
im_host.connection_param[0].username = username
elif header_field == HEADER_FIELD_PASSWORD:
im_host.connection_param[0].password = data_field
elif header_field == HEADER_FIELD_CONNECTION:
im_host.connection_param[0].connection_type = data_field
elif header_field == HEADER_FIELD_PORT:
im_host.connection_param[0].port_number = remove_extra_spaces(data_field)
# Import host already exists in the database, just update it
if db_host is not None:
db_host.created_by = im_host.created_by
db_host.region_id = im_host.region_id
if HEADER_FIELD_ROLES in header_row:
db_host.roles = im_host.roles
if HEADER_FIELD_IP in header_row:
db_host.connection_param[0].host_or_ip = im_host.connection_param[0].host_or_ip
if HEADER_FIELD_USERNAME in header_row:
db_host.connection_param[0].username = im_host.connection_param[0].username
if HEADER_FIELD_PASSWORD in header_row:
db_host.connection_param[0].password = im_host.connection_param[0].password
if HEADER_FIELD_CONNECTION in header_row:
db_host.connection_param[0].connection_type = im_host.connection_param[0].connection_type
if HEADER_FIELD_PORT in header_row:
db_host.connection_param[0].port_number = im_host.connection_param[0].port_number
else:
# Add the import host
db_session.add(im_host)
if error:
return jsonify({'status': error})
else:
db_session.commit()
return jsonify({'status': 'OK'})
class EONet(object):
'''
Object to contain the EONrepr(event['geometries']et events. The event_id, a hashtag, the
geometries and the event title are being retained.
'''
def __init__(self):
self.__session = DBSession().session()
self.__events = self.__session.query(DBEONet).all()
# Update, for good measure.
self.__update_table()
# Add the test event. Needed because it's located in Irvine so the
# geolocation tweets will match.
self.__update_table(TEST_EVENT)
def __update_table(self, json_events=None):
'''
Makes a dictionary of geometries indexed by hashtags. This supports
monitoring the Twitter stream by hashtag and checking the geolocation.
The hashtag is formed from each title by removing punctuation, and then
combining the first three words.
'''
if json_events is None:
eonet = json.load(urllib2.urlopen(EONET_URL))
else:
eonet = json.loads(json_events)
self.events = eonet['events']
print('Requested EONet data: "%s", %d events received.' %
(eonet['description'], len(eonet['events']))
)
# Loop through the events saving them to the database.
for event in self.events:
# Check if the event is already in the database.
eonet_id = self.__session.query(DBEONet).filter(DBEONet.eonet_id == event['id']).one_or_none()
if eonet_id is None:
self.__add_event(event)
def __add_event(self, event):
'''Add an EONet event to the database.'''
# Clean up the title, removing spaces, punctuation and meaningless
# words.
title = event['title']
exclude = set(string.punctuation)
title = ''.join(ch for ch in title if ch not in exclude)
wordlist = title.split()
hashlist = wordlist[:3]
# Assemble the hashtag.
hashtag = '#'
for keyword in hashlist:
hashtag += keyword.capitalize()
# The geometries is a list. The python list object should be,
# so it's pickled and base64 encoded and then reconstituted when it's
# read later.
#
# The base64 version of the list however isn't meaningful to clients
# of the REST API. For that reason, the JSON string representing the
# geometries is also saved.
geometries = PythonDBObject(event['geometries'])
json_geometries = json.dumps(event['geometries'])
new = DBEONet(eonet_id = event['id'],
title = title,
hashtag = hashtag,
geometries = geometries.encode(),
json_geometries = json_geometries
)
self.__session.add(new)
self.__session.commit()
print('Added "%s": %s (%s).' % (new.eonet_id, new.title, new.hashtag))
def refresh(self):
'''
Requests another JSON document of events from EONet and updates the
EONet table in the database.
'''
self.__update_table()
def get_hashtag_from_point(self, point):
'''
Determine if the point either lies within any of the polygon boundaries
or if it's near a point. If yes, return the first the hashtag of the
first matching entry.
The OGR Distance() function returns the distance in the units of the
coordinate system of the geometries, which in the present case are
degree. A degree = 60 nautical miles.
'''
results = self.__session.query(DBEONet.hashtag, DBEONet.geometries)
for hashtag, b64geometries in results:
list_geometries = PythonDBObject(b64geometries).decode()
for dict_geometry in list_geometries:
json_geometry = json.dumps(dict_geometry)
ogr_geometry = ogr.CreateGeometryFromJson(json_geometry)
geotype = ogr_geometry.GetGeometryName()
if 'POLYGON' == geotype:
if point.Within(ogr_geometry):
return hashtag
# Separation is in degrees, which is converted to miles.
if 'POINT' == geotype:
deg_separation = point.Distance(ogr_geometry)
# The international date line can make points look like
# they're almost 360 degrees apart.
if deg_separation > 180:
deg_separation = 360 - deg_separation
sm_separation = degree_to_mile(deg_separation)
if sm_separation < POINT_SEPARATION:
return hashtag
# If nothing's found, return None.
return None
def point_from_hashtag(self, hashtag):
'''
Returns a point given a hashtag.
'''
json_geometry = self.__session.query(DBEOnet.json_geometry).filter(DBEOnet.hashtag == hashtag).one_or_none()
return json_geometry
def records(self):
'''
Returns the complete list of events (eonet_id, title), as a list of
dictionaries.
'''
event_records = []
records = self.__session.query(DBEONet)
for record in records:
event_records.append(record)
return event_records
def hashtags(self):
'''Returns the complete list of the hashtags representing the events.'''
hashtags = []
results = self.__session.query(DBEONet.hashtag)
for result in results:
hashtag = result[0].encode('utf8', 'replace')
hashtags.append(hashtag)
results = self.__session.query(DBEONet.eonet_id)
for result in results:
hashtag = '#'+result[0].encode('utf8', 'replace')
hashtags.append(hashtag)
return hashtags
class SQLDump(TweetConsumer):
'''
Saves the parts of the tweet that the REST API requires to a database.
'''
def __init__(self):
self.__session = DBSession().session()
def _save(self, tweet, hashtag=None):
# Obtain the event from the database.
event = self.__session.query(DBEONet).filter(DBEONet.hashtag == hashtag).one_or_none()
# No need to continue if the tweet is already in the database.
db_tweet = self.__session.query(DBTweets).filter(DBTweets.tweet_id == tweet.id_str).one_or_none()
if db_tweet is not None:
return
# the data fields to save.
name = tweet.user.name
create_date = tweet.created_at
place = tweet.place.name if tweet.place else None
text = tweet.text
profile_pic = tweet.user.profile_image_url
screen_name = tweet.user.screen_name
try:
tweet.extended_entities
except AttributeError:
media_url = None
else:
media_url = tweet.extended_entities['media'][0]['media_url'] if tweet.extended_entities['media'] and tweet.extended_entities['media'][0]['type'] == 'photo' else None
# The coordinates of the tweet are either the specific long, lat
# provided in the tweet, or the centroid of the place's bounding
# box.
#
# The coordinates is a list that must be saved as a string. However
# the list needs to be reconstituted when it's read. The clients
# consuming the REST API need the JSON version of the coordinates.
if tweet.coordinates is not None:
coordinates = PythonDBObject(tweet.coordinates)
json_coordinates = json.dumps(tweet.coordinates)
# Second choice is the centroid of the place
elif tweet.place is not None:
center = place_centroid(tweet)
# Build the dictionary.
l_coords = [center.GetX(), center.GetY()]
d_coords = {u'type': 'Point', u'coordinates': l_coords}
coordinates = PythonDBObject(d_coords)
json_coordinates = json.dumps(d_coords)
# Use the centroid of the event if we have that.
elif hashtag is not None:
e_json = event.json_geometries
f_json = str(e_json)
g_json = f_json[1:-1]
ogr_geometry = ogr.CreateGeometryFromJson(g_json)
geotype = ogr_geometry.GetGeometryName
if 'Point' == geotype:
coordinates = PythonDBObject(g_json)
json_coordinates =g_json
else:
center = ogr_geometry.Centroid()
# Build the dictionary.
l_coords = [center.GetX(), center.GetY()]
d_coords = {u'type': 'Point', u'coordinates': l_coords}
coordinates = PythonDBObject(d_coords)
json_coordinates = json.dumps(d_coords)
# Commit to the database.
tweet_record = DBTweets(tweet_id=tweet.id_str,
eonet_id=event.eonet_id,
name=name,
createDate=create_date,
place=place,
msg=text,
coordinates=coordinates.encode(),
hashtag=hashtag,
media_url=media_url,
screen_name=screen_name,
profile_pic=profile_pic,
json_coordinates=json_coordinates)
self.__session.add(tweet_record)
self.__session.commit()
# Write the same to standard out to monitor the progress. The terminal
# requires the text to be encoded as utf8.
text = text.encode('utf8', 'replace')
name = name.encode('utf8', 'replace')
place = place.encode('utf8', 'replace') if place else None
media_url = media_url.encode('utf8', 'replace') if media_url else None
print('Written to the database: [Name: %s][Place: %s][At: %s] %s %s.' %
(name, place, create_date, text, media_url))
def init_system_option():
db_session = DBSession()
if db_session.query(SystemOption).count() == 0:
db_session.add(SystemOption())
db_session.commit()
