def user_create(): if not can_create_user(current_user): abort(401) form = UserForm(request.form) # Need to add the Required flag back as it is globally removed during user_edit() add_validator(form.password, Required) fill_user_privileges(form.privilege.choices) if request.method == 'POST' and form.validate(): db_session = DBSession() user = get_user(db_session, form.username.data) if user is not None: return render_template('user/edit.html', form=form, duplicate_error=True) user = User( username=form.username.data, password=form.password.data, privilege=form.privilege.data, fullname=form.fullname.data, email=form.email.data) user.preferences.append(Preferences()) db_session.add(user) db_session.commit() return redirect(url_for('home')) else: # Default to Active form.active.data = True return render_template('user/edit.html', form=form)
def refresh_all(cls): """ Retrieves all the catalog data and SMU XML file data and updates the database. """ db_session = DBSession() catalog = SMUInfoLoader.get_catalog_from_cco() if len(catalog) > 0: system_option = SystemOption.get(db_session) try: # Remove all rows first db_session.query(CCOCatalog).delete() for platform in catalog: releases = catalog[platform] for release in releases: cco_catalog = CCOCatalog(platform=platform,release=release) db_session.add(cco_catalog) SMUInfoLoader(platform, release) system_option.cco_lookup_time = datetime.datetime.utcnow() db_session.commit() return True except Exception: logger.exception('refresh_all() hit exception') db_session.rollback() return False
def _collect_video_info(self,tid, simple_info, session:DBSession, collect_id = 0): uploader_exists = session.query( exists().where(database.UploaderInfo.mid == simple_info['author_id']) ).scalar() if not uploader_exists: new_up = database.UploaderInfo( mid=simple_info['author_id'], name=simple_info['author_name'], ) session.add(new_up) session.commit() new_video = database.VideoInfo( aid=simple_info['id'], bvid=simple_info['bvid'], tid=tid, title=simple_info['name'], owner_id=simple_info['author_id'], collect_time=datetime.datetime.now(), collect_id = collect_id ) try: session.add(new_video) session.commit() except Exception as e: logger.error(f"向数据库中写入视频信息出错: {simple_info['name']}") session.rollback()
def api_create_tar_job(): db_session = DBSession() form = CreateTarForm(request.form) server_id = request.args.get('server') server_directory = request.args.get('server_directory') source_tars = request.args.getlist('source_tars[]') contents = request.args.getlist('tar_contents[]') additional_packages = request.args.getlist('additional_packages[]') new_tar_name = request.args.get('new_tar_name').strip('.tar') create_tar_job = CreateTarJob( server_id = server_id, server_directory = server_directory, source_tars = (',').join(source_tars), contents = (',').join(contents), additional_packages = (',').join(additional_packages), new_tar_name = new_tar_name, created_by = current_user.username, status = 'Job Submitted.') db_session.add(create_tar_job) db_session.commit() job_id = create_tar_job.id return jsonify({'status': 'OK', 'job_id': job_id})
def command_profile_create(): db_session = DBSession() form = CustomCommandProfileForm(request.form) if request.method == 'POST' and form.validate(): command_profile = get_command_profile(db_session, form.profile_name.data) if command_profile is not None: return render_template('custom_command/command_profile_edit.html', form=form, duplicate_error=True) command_profile = CustomCommandProfile( profile_name=form.profile_name.data, command_list=','.join( [l for l in form.command_list.data.splitlines() if l]), created_by=current_user.username) db_session.add(command_profile) db_session.commit() return redirect(url_for('custom_command.home')) else: return render_template('custom_command/command_profile_edit.html', form=form)
def filesupload(): if request.method == 'POST': file = request.files['file'] if file: session = DBSession() filename = datetime.datetime.now().strftime( "%Y%m%d%H%M%S_") + secure_filename(file.filename) user = current_user._get_current_object() filepath = os.path.join(os.getcwd(), 'static', 'files', '{}'.format(user.userid)) # now=time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(int(time.time()))) now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") print(now) print(filepath) # file.save(os.path.join(os.getcwd(),'static',filename)) file.save(os.path.join(filepath, filename)) filerec = files(filename=filename, filepath=filepath, user_userid=user.userid, uploadtime=now, quote=1) session.add(filerec) session.commit() record = filerecord(user_userid=user.userid, files_fileid=filerec.fileid, flag=1, recordtime=now) session.add(record) session.commit() session.close() return 'File {0} has benn saved in {1} !'.format( filename, filepath) else: return render_template('uploadtest.html')
def command_profile_create(): db_session = DBSession() form = CustomCommandProfileForm(request.form) if request.method == 'POST' and form.validate(): command_profile = get_command_profile(db_session, form.profile_name.data) if command_profile is not None: return render_template('custom_command/command_profile_edit.html', form=form, duplicate_error=True) command_profile = CustomCommandProfile( profile_name=form.profile_name.data, command_list=','.join([l for l in form.command_list.data.splitlines() if l]), created_by=current_user.username ) db_session.add(command_profile) db_session.commit() return redirect(url_for('custom_command.home')) else: return render_template('custom_command/command_profile_edit.html', form=form)
def refresh_all(cls): """ Retrieves all the catalog data and SMU XML file data and updates the database. """ db_session = DBSession() catalog = SMUInfoLoader.get_catalog_from_cco() if len(catalog) > 0: system_option = SystemOption.get(db_session) try: # Remove all rows first db_session.query(CCOCatalog).delete() for platform in catalog: releases = catalog[platform] for release in releases: cco_catalog = CCOCatalog(platform=platform, release=release) db_session.add(cco_catalog) # Now, retrieve from SMU XML file SMUInfoLoader(platform, release, refresh=True) system_option.cco_lookup_time = datetime.datetime.utcnow() db_session.commit() return True except Exception: logger.exception('refresh_all hit exception') db_session.rollback() return False
def software_profile_create(): # if not can_create_user(current_user): # abort(401) db_session = DBSession() form = SoftwareProfileForm(request.form) server_dialog_form = ServerDialogForm(request.form) fill_servers(server_dialog_form.server_dialog_server.choices, get_server_list(db_session), False) if request.method == 'POST' and form.validate(): software_profile = get_software_profile(db_session, form.profile_name.data) if software_profile is not None: return render_template('conformance/profile_edit.html', form=form, system_option=SystemOption.get(db_session), duplicate_error=True) software_profile = SoftwareProfile( name=form.profile_name.data, description=form.description.data, packages=','.join([l for l in form.software_packages.data.splitlines() if l]), created_by=current_user.username) db_session.add(software_profile) db_session.commit() return redirect(url_for('conformance.home')) else: return render_template('conformance/profile_edit.html', form=form, server_dialog_form=server_dialog_form, system_option=SystemOption.get(db_session))
def insert_messages(messages): from database import DBSession, Message, User for index, message in enumerate(messages): new_msg = Message(id=-index, link='', text='{}: {}'.format(message.sender, message.message), video='', photo='', audio='', voice='', type='text', category='', from_id=0, date=message.date) session = DBSession() session.add(new_msg) session.commit() session.close() deafult_user = User(id=0, fullname='历史聊天记录', username='******', update_time=datetime.now()) session = DBSession() session.add(deafult_user) session.commit() session.close()
def userregister(): if request.method == 'POST': session = DBSession() email = request.form['useremail'] username = request.form['username'] password = request.form['userpassword'] password = generate_password_hash(password) print(email) if session.query(User).filter(User.email == email).first(): return render_template('register.html', msg='该邮箱已被注册!') else: adduser = User(password=password, username=username, email=email, admin=0,activated=0) session.add(adduser) session.commit() os.makedirs(os.path.join(os.getcwd(),'static','files','{0}'.format(adduser.userid))) url='http://47.94.138.25/activate/{0}'.format(gen_token(adduser.userid)) # message=Message('Account authorize',sender='*****@*****.**',recipients=['{}'.format(username),'{}'.format(email)]) message = Message('Account authorize', sender='*****@*****.**',recipients=['*****@*****.**']) message.body="Hello,{0}<br>This email is from admin!<br>Your account has not been activated yet!<br><a href='{1}'>click here to activate your account!</a><br>if you mistaken the email,please ignore it!".format(username,url) mail.send(message) session.close() return redirect(url_for('users.userlogin')) else: return render_template('register.html')
def insert_messages(messages): from database import DBSession, Message, User for message in messages: new_msg = Message(id=message.msgid, link='https://t.me/c/{}/{}'.format( LINK_ID, message.msgid), text=message.message, video='', photo='', audio='', voice='', type='text', category='', from_id=message.senderid, date=message.date) session = DBSession() session.add(new_msg) if not message.userexist: new_user = User(id=message.senderid, fullname=message.sender, username='', update_time=datetime.now()) session.add(new_user) session.commit() session.close()
def api_create_software_profile(): profile_name = request.form['profile_name'] description = request.form['description'] software_packages = request.form['software_packages'] db_session = DBSession() software_profile = get_software_profile(db_session, profile_name) if software_profile is not None: return jsonify({ 'status': 'Software profile "' + profile_name + '" already exists. Use a different name instead.' }) software_profile = SoftwareProfile( name=profile_name, description=description, packages=','.join([l for l in software_packages.splitlines() if l]), created_by=current_user.username) db_session.add(software_profile) db_session.commit() return jsonify({'status': 'OK'})
def api_create_tar_job(): db_session = DBSession() form = CreateTarForm(request.form) server_id = request.args.get('server') server_directory = request.args.get('server_directory') source_tars = request.args.getlist('source_tars[]') contents = request.args.getlist('tar_contents[]') additional_packages = request.args.getlist('additional_packages[]') new_tar_name = request.args.get('new_tar_name').replace('.tar', '') create_tar_job = CreateTarJob( server_id=server_id, server_directory=server_directory, source_tars=','.join(source_tars), contents=','.join(contents), additional_packages=','.join(additional_packages), new_tar_name=new_tar_name, created_by=current_user.username, status='Job Submitted.') db_session.add(create_tar_job) db_session.commit() job_id = create_tar_job.id return jsonify({'status': 'OK', 'job_id': job_id})
def user_create(): if not can_create_user(current_user): abort(401) form = UserForm(request.form) # Need to add the Required flag back as it is globally removed during user_edit() add_validator(form.password, Required) fill_user_privileges(form.privilege.choices) if request.method == 'POST' and form.validate(): db_session = DBSession() user = get_user(db_session, form.username.data) if user is not None: return render_template('user/edit.html', form=form, duplicate_error=True) user = User(username=form.username.data, password=form.password.data, privilege=form.privilege.data, fullname=form.fullname.data, email=form.email.data) user.preferences.append(Preferences()) db_session.add(user) db_session.commit() return redirect(url_for('home')) else: # Default to Active form.active.data = True return render_template('user/edit.html', form=form)
def get_smu_info_from_cco(self, platform, release): save_to_db = True db_session = DBSession() platform_release = platform + '_' + release try: self.smu_meta = SMUMeta(platform_release=platform_release) # Load data from the SMU XML file self.load() # This can happen if the given platform and release is not valid. # The load() method calls get_smu_info_from_db and failed. if not self.is_valid: logger.error('get_smu_info_from_cco() hit exception, platform_release=' + platform_release) return db_smu_meta = db_session.query(SMUMeta).filter(SMUMeta.platform_release == platform_release).first() if db_smu_meta: if db_smu_meta.created_time == self.smu_meta.created_time: save_to_db = False else: # Delete the existing smu_meta and smu_info for this platform and release db_session.delete(db_smu_meta) db_session.commit() if save_to_db: db_session.add(self.smu_meta) else: db_smu_meta.retrieval_time = datetime.datetime.utcnow() db_session.commit() except Exception: logger.exception('get_smu_info_from_cco() hit exception, platform_release=' + platform_release)
def init(): session = DBSession() if session.query(Config).filter().count() == 0: user = USER + ':' + PASSWORD movie_dir_re = os.environ['MOVIE_DIR_RE'] tg_push_on = True if os.environ['TG_ON'] == 'true' else False tg_chatid = os.environ['TG_CHATID'] tg_bot_token = os.environ['TG_BOT_TOKEN'] bark_push_on = True if os.environ['BARK_ON'] == 'true' else False bark_tokens = os.environ['BARK_TOKENS'] server_cyann_on = True if os.environ[ 'SERVER_CYANN'] == 'true' else False server_cyann_token = os.environ['SERVER_CYANN_TOKEN'] proxy_on = False proxy_url = '' config = Config(user=user, root_dir='/mnt/media', movie_dir_re=movie_dir_re, tg_push_on=tg_push_on, tg_chatid=tg_chatid, tg_bot_token=tg_bot_token, bark_push_on=bark_push_on, bark_tokens=bark_tokens, server_cyann_on=server_cyann_on, server_cyann_token=server_cyann_token, proxy_on=proxy_on, proxy_url=proxy_url) session.add(config) session.commit() session.close()
def insert_user_or_do_nothing(user_id, fullname, username): session = DBSession() target_user = session.query(User).get(user_id) if not target_user: new_user = User(id=user_id, fullname=fullname, username=username) session.add(new_user) session.commit() session.close()
def insert_chat_or_do_nothing(chat_id, title): session = DBSession() target_chat = session.query(Chat).get(chat_id) if not target_chat: new_chat = Chat(id=chat_id, title=title, enable=False) session.add(new_chat) session.commit() session.close()
def init_encrypt(): global encrypt_dict db_session = DBSession() if db_session.query(Encrypt).count() == 0: db_session.add(Encrypt()) db_session.commit() encrypt_dict = dict(Encrypt.get(db_session).__dict__)
def init_encrypt(): global encrypt_dict db_session = DBSession() if db_session.query(Encrypt).count() == 0: db_session.add(Encrypt()) db_session.commit() encrypt_dict = dict(Encrypt.get(db_session).__dict__)
def init_sys_time(): db_session = DBSession() if db_session.query(System).count() == 0: db_session.add(System()) db_session.commit() else: system = db_session.query(System).first() system.start_time = datetime.datetime.utcnow() db_session.commit()
def init_sys_time(): db_session = DBSession() if db_session.query(System).count() == 0: db_session.add(System()) db_session.commit() else: system = db_session.query(System).first() system.start_time = datetime.datetime.utcnow() db_session.commit()
def insert_or_update_user(user_id, fullname, username): session = DBSession() target_user = session.query(User).get(user_id) if not target_user: new_user = User(id=user_id, fullname=fullname, username=username) session.add(new_user) session.commit() elif target_user.fullname != fullname or target_user.username != username: target_user.fullname = fullname target_user.username = username session.commit() session.close()
def disbale_chat_or_do_nothing(chat_id): session = DBSession() target_chat = session.query(Chat).get(chat_id) if target_chat and target_chat.enable: target_chat.enable = False session.add(target_chat) session.commit() msg_text = '成功停用,记录暂时保留,删除需使用/delete命令' else: msg_text = '此前已停用/未启用,启用需使用/start命令' session.close() return msg_text
def insert_messages(chat_id, messages): fail_count = 0 fail_messages = [] success_count = 0 for message in messages: if 'from_id' not in message: continue elif 'user' not in message['from_id']: continue insert_user_or_do_nothing(message['from_id'][4:], message['from'], message['from']) if isinstance(message['text'], list): msg_text = '' for obj in message['text']: if isinstance(obj, dict): msg_text += obj['text'] else: msg_text += obj else: msg_text = message['text'] if msg_text == '': msg_text == '[其他消息]' message_date = datetime.strptime(message['date'], '%Y-%m-%dT%H:%M:%S') new_msg = Message(id=message['id'], link='https://t.me/c/{}/{}'.format( chat_id, message['id']), text=msg_text, video='', photo='', audio='', voice='', type='text', category='', from_id=message['from_id'][4:], from_chat=chat_id, date=message_date) session = DBSession() try: session.add(new_msg) session.commit() success_count += 1 except Exception as e: print(e) fail_count += 1 fail_messages.append(str(message)) session.close() return success_count, fail_count, fail_messages
def insert_message(msg_id, msg_text, msg_video, msg_photo, msg_audio, msg_voice, msg_type, from_id, date): new_msg = Message(id=msg_id, text=msg_text, video=msg_video, photo=msg_photo, audio=msg_audio, voice=msg_voice, type=msg_type, category='', from_id=from_id, date=date) session = DBSession() session.add(new_msg) session.commit() session.close()
def useraccfriend(userid): if request.method == 'GET': session = DBSession() user = current_user._get_current_object() if userid == user.userid: return Response(response='Failed!', status=400) else: now=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) userre = userrelationship(user=user.userid, friend=userid, flag=1,rtime=now) session.add(userre) fr=session.query(userrelationship).filter(userrelationship.user == userid, userrelationship.friend == user.userid).first() fr.flag=1 fr.rtime=now session.commit() session.close() return Response(response='successfully!', status=200)
def insert_chat_or_enable(chat_id, title): session = DBSession() target_chat = session.query(Chat).get(chat_id) if not target_chat: new_chat = Chat(id=chat_id, title=title, enable=True) session.add(new_chat) session.commit() msg_text = '成功启用' else: if target_chat.enable: msg_text = '前期已启用,停用需使用/stop命令' else: target_chat.enable = True session.commit() msg_text = '成功恢复使用' session.close() return msg_text
def convert_config_file(): filename = request.args.get('filename', '', type=str) filename = secure_filename(filename) config_conversion_path = get_config_conversion_path() db_session = DBSession() convert_config_job = ConvertConfigJob(file_path=os.path.join(config_conversion_path, filename), status='Preparing the conversion') db_session.add(convert_config_job) db_session.commit() job_id = convert_config_job.id return jsonify({'status': 'OK', 'job_id': job_id})
def convert_config_file(): filename = request.args.get('filename', '', type=str) filename = secure_filename(filename) config_conversion_path = get_config_conversion_path() db_session = DBSession() convert_config_job = ConvertConfigJob(file_path=os.path.join( config_conversion_path, filename), status='Preparing the conversion') db_session.add(convert_config_job) db_session.commit() job_id = convert_config_job.id return jsonify({'status': 'OK', 'job_id': job_id})
def emit(self, record): trace = traceback.format_exc() if record.__dict__['exc_info'] else None args = record.__dict__['args'] msg = record.__dict__['msg'] if len(args) >= 1: msg = msg % args log = Log(level=record.__dict__['levelname'], trace=trace, msg=msg, created_time=datetime.datetime.utcnow()) db_session = DBSession() db_session.add(log) db_session.commit()
def emit(self, record): trace = traceback.format_exc() if record.__dict__['exc_info'] else None args = record.__dict__['args'] msg = record.__dict__['msg'] if len(args) >= 1: msg = msg % args log = Log( level=record.__dict__['levelname'], trace=trace, msg=msg, created_time=datetime.datetime.utcnow()) db_session = DBSession() db_session.add(log) db_session.commit()
def write_to_database(self): """ 将CheckUpdate实例的info_dic数据写入数据库 """ session = DBSession() try: if self.name in {x.ID for x in session.query(Saved).all()}: saved_data = session.query(Saved).filter(Saved.ID == self.name).one() saved_data.FULL_NAME = self.fullname for key, value in self.__info_dic.items(): setattr(saved_data, key, value) else: new_data = Saved( ID=self.name, FULL_NAME=self.fullname, **self.__info_dic ) session.add(new_data) session.commit() finally: session.close()
def useraddfriend(email): if request.method == 'GET': session = DBSession() user = current_user._get_current_object() friend=session.query(User).filter(User.email == email).first() userid=friend.userid if session.query(userrelationship).filter(userrelationship.user == user.userid, userrelationship.friend == userid).first(): # user has been send request return Response(response='You have been send request or already friend!', status=400) if userid == user.userid: return Response(response='Failed!', status=400) else: # now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) userre = userrelationship(user=user.userid, friend=userid, flag=0,rtime=now) session.add(userre) session.commit() session.close() return Response(response='successfully!', status=200)
def get_database_file(bot, update): if update.message.chat_id != config.GROUP_ID: return delete_prev_message(bot, update) now = datetime.datetime.now() session = DBSession() count = session.query(DBFile).filter().count() if count: last_db_file = session.query(DBFile).filter().order_by( DBFile.date.desc()).first() else: last_db_file = None if last_db_file and last_db_file.date + datetime.timedelta( minutes=10) > now: file = last_db_file.file_id filename = 'bot.{}.db'.format( last_db_file.date.strftime("%Y-%m-%d_%H.%M")) new_record = False else: file = open('bot.db', 'rb') filename = 'bot.{}.db'.format(now.strftime("%Y-%m-%d_%H.%M")) new_record = True sent_message = bot.send_document(chat_id=update.message.chat_id, document=file, filename=filename, disable_notification=True) # 保存新的获取数据库记录 if new_record: file_id = sent_message.document.file_id new_db_file = DBFile(file_id=file_id, date=now) session.add(new_db_file) session.commit() session.close() return sent_message
def software_profile_create(): # if not can_create_user(current_user): # abort(401) db_session = DBSession() form = SoftwareProfileForm(request.form) server_dialog_form = ServerDialogForm(request.form) fill_servers(server_dialog_form.server_dialog_server.choices, get_server_list(db_session), False) if request.method == 'POST' and form.validate(): software_profile = get_software_profile(db_session, form.profile_name.data) if software_profile is not None: return render_template('conformance/profile_edit.html', form=form, system_option=SystemOption.get(db_session), duplicate_error=True) software_profile = SoftwareProfile( name=form.profile_name.data, description=form.description.data, packages=','.join( [l for l in form.software_packages.data.splitlines() if l]), created_by=current_user.username) db_session.add(software_profile) db_session.commit() return redirect(url_for('conformance.home')) else: return render_template('conformance/profile_edit.html', form=form, server_dialog_form=server_dialog_form, system_option=SystemOption.get(db_session))
def get_smu_info_from_cco(self, platform, release): same_as_db = False db_session = DBSession() platform_release = platform + '_' + release try: self.smu_meta = SMUMeta(platform_release=platform_release) # Load data from the SMU XML file self.load() # This can happen if the given platform and release is not valid. # The load method calls get_smu_info_from_db and failed. if self.smu_meta is None: return db_smu_meta = db_session.query(SMUMeta).filter( SMUMeta.platform_release == platform + '_' + release).first() if db_smu_meta is not None: if db_smu_meta.created_time == self.smu_meta.created_time: same_as_db = True else: # Delete the existing smu_meta and smu_info for this platform and release db_session.delete(db_smu_meta) if not same_as_db: db_session.add(self.smu_meta) else: db_smu_meta.retrieval_time = datetime.datetime.utcnow() # Use Flush to detect concurrent saving condition. It is # possible that another process may perform the same save. # If this happens, Duplicate Key may result. db_session.flush() db_session.commit() except IntegrityError: db_session.rollback() except Exception: db_session.rollback() logger.exception('get_smu_info_from_cco hit exception')
def sendfile(recuserid, fileid, note): session = DBSession() senduser = current_user._get_current_object() print(senduser) print(recuserid, fileid, note) now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") if note is None or note == 'None': note = '00' if session.query(User).filter(User.userid == recuserid).first( ) and session.query(filerecord).filter( filerecord.files_fileid == fileid, filerecord.flag == 1, filerecord.user_userid == senduser.userid).first() and recuserid != senduser.userid and ( session.query(userrelationship).filter( userrelationship.user == recuserid, userrelationship.friend == senduser.userid).first() and session.query(userrelationship).filter( userrelationship.user == senduser.userid, userrelationship.friend == recuserid).first()): src = sendrecord(user_userid=senduser.userid, user_userid1=recuserid, files_fileid=fileid, sendtime=now, note=note) session.add(src) session.commit() recfile = filerecord(user_userid=recuserid, files_fileid=fileid, flag=1, recordtime=now) session.add(recfile) session.commit() filequote = session.query(files).filter(files.fileid == fileid).first() filequote.quote += 1 filename = filequote.filename session.commit() session.close() return 'send file:{0} successfully!'.format(filename) else: return 'send fail!'
def get_smu_info_from_cco(self, platform, release): save_to_db = True db_session = DBSession() platform_release = platform + '_' + release try: self.smu_meta = SMUMeta(platform_release=platform_release) # Load data from the SMU XML file self.load() # This can happen if the given platform and release is not valid. # The load() method calls get_smu_info_from_db and failed. if not self.is_valid: logger.error( 'get_smu_info_from_cco() hit exception, platform_release=' + platform_release) return db_smu_meta = db_session.query(SMUMeta).filter( SMUMeta.platform_release == platform_release).first() if db_smu_meta: if db_smu_meta.created_time == self.smu_meta.created_time: save_to_db = False else: # Delete the existing smu_meta and smu_info for this platform and release db_session.delete(db_smu_meta) db_session.commit() if save_to_db: db_session.add(self.smu_meta) else: db_smu_meta.retrieval_time = datetime.datetime.utcnow() db_session.commit() except Exception: logger.exception( 'get_smu_info_from_cco() hit exception, platform_release=' + platform_release)
def get_smu_info_from_cco(self, platform, release): same_as_db = False db_session = DBSession() platform_release = platform + '_' + release try: self.smu_meta = SMUMeta(platform_release=platform_release) # Load data from the SMU XML file self.load() # This can happen if the given platform and release is not valid. # The load method calls get_smu_info_from_db and failed. if self.smu_meta is None: return db_smu_meta = db_session.query(SMUMeta).filter(SMUMeta.platform_release == platform + '_' + release).first() if db_smu_meta is not None: if db_smu_meta.created_time == self.smu_meta.created_time: same_as_db = True else: # Delete the existing smu_meta and smu_info for this platform and release db_session.delete(db_smu_meta) if not same_as_db: db_session.add(self.smu_meta) else: db_smu_meta.retrieval_time = datetime.datetime.utcnow() # Use Flush to detect concurrent saving condition. It is # possible that another process may perform the same save. # If this happens, Duplicate Key may result. db_session.flush() db_session.commit() except IntegrityError: db_session.rollback() except Exception: db_session.rollback() logger.exception('get_smu_info_from_cco() hit exception')
def api_create_software_profile(): profile_name = request.form['profile_name'] description = request.form['description'] software_packages = request.form['software_packages'] db_session = DBSession() software_profile = get_software_profile(db_session, profile_name) if software_profile is not None: return jsonify({'status': 'Software profile "' + profile_name + '" already exists. Use a different name instead.'}) software_profile = SoftwareProfile( name=profile_name, description=description, packages=','.join([l for l in software_packages.splitlines() if l]), created_by=current_user.username) db_session.add(software_profile) db_session.commit() return jsonify({'status': 'OK'})
def home(): if current_user.privilege != UserPrivilege.ADMIN: abort(401) db_session = DBSession() smtp_form = SMTPForm(request.form) admin_console_form = AdminConsoleForm(request.form) smtp_server = get_smtp_server(db_session) system_option = SystemOption.get(db_session) fill_user_privileges(admin_console_form.ldap_default_user_privilege.choices) if request.method == 'POST' and \ smtp_form.validate() and \ admin_console_form.validate(): if smtp_server is None: smtp_server = SMTPServer() db_session.add(smtp_server) smtp_server.server = smtp_form.server.data smtp_server.server_port = smtp_form.server_port.data if len(smtp_form.server_port.data) > 0 else None smtp_server.sender = smtp_form.sender.data smtp_server.use_authentication = smtp_form.use_authentication.data smtp_server.username = smtp_form.username.data if len(smtp_form.password.data) > 0: smtp_server.password = smtp_form.password.data smtp_server.secure_connection = smtp_form.secure_connection.data system_option.inventory_threads = admin_console_form.num_inventory_threads.data system_option.install_threads = admin_console_form.num_install_threads.data system_option.download_threads = admin_console_form.num_download_threads.data system_option.can_schedule = admin_console_form.can_schedule.data system_option.can_install = admin_console_form.can_install.data system_option.enable_email_notify = admin_console_form.enable_email_notify.data system_option.enable_inventory = admin_console_form.enable_inventory.data # The LDAP UI may be hidden if it is not supported. # In this case, the flag is not set. if not is_empty(admin_console_form.enable_ldap_auth.data): system_option.enable_ldap_auth = admin_console_form.enable_ldap_auth.data system_option.ldap_server_url = admin_console_form.ldap_server_url.data system_option.ldap_default_user_privilege = admin_console_form.ldap_default_user_privilege.data system_option.ldap_server_distinguished_names = admin_console_form.ldap_server_distinguished_names.data.strip() system_option.inventory_hour = admin_console_form.inventory_hour.data system_option.inventory_history_per_host = admin_console_form.inventory_history_per_host.data system_option.download_history_per_user = admin_console_form.download_history_per_user.data system_option.install_history_per_host = admin_console_form.install_history_per_host.data system_option.total_system_logs = admin_console_form.total_system_logs.data system_option.enable_default_host_authentication = admin_console_form.enable_default_host_authentication.data system_option.default_host_authentication_choice = admin_console_form.default_host_authentication_choice.data system_option.enable_cco_lookup = admin_console_form.enable_cco_lookup.data system_option.use_utc_timezone = admin_console_form.use_utc_timezone.data system_option.default_host_username = admin_console_form.default_host_username.data if len(admin_console_form.default_host_password.data) > 0: system_option.default_host_password = admin_console_form.default_host_password.data system_option.enable_user_credential_for_host = admin_console_form.enable_user_credential_for_host.data db_session.commit() return redirect(url_for('home')) else: admin_console_form.num_inventory_threads.data = system_option.inventory_threads admin_console_form.num_install_threads.data = system_option.install_threads admin_console_form.num_download_threads.data = system_option.download_threads admin_console_form.can_schedule.data = system_option.can_schedule admin_console_form.can_install.data = system_option.can_install admin_console_form.enable_email_notify.data = system_option.enable_email_notify admin_console_form.enable_ldap_auth.data = system_option.enable_ldap_auth admin_console_form.ldap_server_url.data = system_option.ldap_server_url admin_console_form.ldap_default_user_privilege.data = system_option.ldap_default_user_privilege admin_console_form.ldap_server_distinguished_names.data = system_option.ldap_server_distinguished_names admin_console_form.enable_inventory.data = system_option.enable_inventory admin_console_form.inventory_hour.data = system_option.inventory_hour admin_console_form.inventory_history_per_host.data = system_option.inventory_history_per_host admin_console_form.download_history_per_user.data = system_option.download_history_per_user admin_console_form.install_history_per_host.data = system_option.install_history_per_host admin_console_form.total_system_logs.data = system_option.total_system_logs admin_console_form.enable_default_host_authentication.data = system_option.enable_default_host_authentication admin_console_form.default_host_authentication_choice.data = system_option.default_host_authentication_choice admin_console_form.default_host_username.data = system_option.default_host_username admin_console_form.enable_cco_lookup.data = system_option.enable_cco_lookup admin_console_form.use_utc_timezone.data = system_option.use_utc_timezone admin_console_form.cco_lookup_time.data = get_datetime_string(system_option.cco_lookup_time) admin_console_form.enable_user_credential_for_host.data = system_option.enable_user_credential_for_host if not is_empty(system_option.default_host_password): admin_console_form.default_host_password_placeholder = 'Use Password on File' else: admin_console_form.default_host_password_placeholder = 'No Password Specified' if smtp_server is not None: smtp_form.server.data = smtp_server.server smtp_form.server_port.data = smtp_server.server_port smtp_form.sender.data = smtp_server.sender smtp_form.use_authentication.data = smtp_server.use_authentication smtp_form.username.data = smtp_server.username smtp_form.secure_connection.data = smtp_server.secure_connection if not is_empty(smtp_server.password): smtp_form.password_placeholder = 'Use Password on File' else: smtp_form.password_placeholder = 'No Password Specified' return render_template('admin/index.html', admin_console_form=admin_console_form, smtp_form=smtp_form, system_option=SystemOption.get(db_session), is_ldap_supported=is_ldap_supported())
def api_import_hosts(): region_id = int(request.form['region_id']) jump_host_id = int(request.form['jump_host_id']) software_profile_id = int(request.form['software_profile_id']) data_list = request.form['data_list'] db_session = DBSession() if region_id == -1: return jsonify({'status': 'Region has not been specified.'}) if region_id > 0: region = get_region_by_id(db_session, region_id) if region is None: return jsonify({'status': 'Region is no longer exists in the database.'}) if jump_host_id > 0: jump_host = get_jump_host_by_id(db_session, jump_host_id) if jump_host is None: return jsonify({'status': 'Jump Host is no longer exists in the database.'}) if software_profile_id > 0: software_profile = get_software_profile_by_id(db_session, software_profile_id) if software_profile is None: return jsonify({'status': 'Software Profile is no longer exists in the database.'}) error = [] reader = csv.reader(data_list.splitlines(), delimiter=',') header_row = next(reader) # header_row: ['hostname', 'location', 'roles', 'ip', 'username', 'password', 'connection', 'port'] # Check mandatory data fields if HEADER_FIELD_HOSTNAME not in header_row: error.append('"hostname" is missing in the header.') if HEADER_FIELD_IP not in header_row: error.append('"ip" is missing in the header.') if HEADER_FIELD_CONNECTION not in header_row: error.append('"connection" is missing in the header.') for header_field in header_row: if header_field not in HEADER_FIELDS: error.append('"' + header_field + '" is not a correct header field.') if error: return jsonify({'status': '\n'.join(error)}) error = [] data_list = list(reader) region_dict = get_region_name_to_id_dict(db_session) # Check if each row has the same number of data fields as the header row = 2 for row_data in data_list: if len(row_data) != len(header_row): error.append('line {} has wrong number of data fields - {}.'.format(row, row_data)) else: hostname = get_acceptable_string(get_row_data(row_data, header_row, HEADER_FIELD_HOSTNAME)) if is_empty(hostname): error.append('line {} has invalid hostname - {}.'.format(row, row_data)) # Validate the connection type connection_type = get_row_data(row_data, header_row, HEADER_FIELD_CONNECTION) if is_empty(connection_type) or connection_type not in [ConnectionType.TELNET, ConnectionType.SSH]: error.append('line {} has a wrong connection type (should either be "telnet" or "ssh") - {}.'.format(row, row_data)) region_name = get_acceptable_string(get_row_data(row_data, header_row, HEADER_FIELD_REGION)) if region_name is not None: # No blank region is allowed if len(region_name) == 0: error.append('line {} has no region specified - {}.'.format(row, row_data)) else: if region_name not in region_dict.keys(): # Create the new region try: region = Region(name=region_name, created_by=current_user.username) db_session.add(region) db_session.commit() # Add to region dictionary for caching purpose. region_dict[region_name] = region.id except Exception as e: logger.exception('api_import_hosts() hit exception') error.append('Unable to create region {} - {}.'.format(region_name, e.message)) row += 1 if error: return jsonify({'status': '\n'.join(error)}) # Import the data row = 2 for row_data in data_list: try: created_by = current_user.username hostname = get_acceptable_string(get_row_data(row_data, header_row, HEADER_FIELD_HOSTNAME)) # Check if the host already exists in the database. host = get_host(db_session, hostname) region_name = get_acceptable_string(get_row_data(row_data, header_row, HEADER_FIELD_REGION)) if region_name is None: alternate_region_id = region_id else: alternate_region_id = region_dict[region_name] location = get_row_data(row_data, header_row, HEADER_FIELD_LOCATION) if host and location is None: location = host.location roles = get_row_data(row_data, header_row, HEADER_FIELD_ROLES) if host and roles is None: roles = host.roles host_or_ip = get_row_data(row_data, header_row, HEADER_FIELD_IP) if host and host_or_ip is None: host_or_ip = host.connection_param[0].host_or_ip connection_type = get_row_data(row_data, header_row, HEADER_FIELD_CONNECTION) if host and connection_type is None: connection_type = host.connection_param[0].connection_type username = get_row_data(row_data, header_row, HEADER_FIELD_USERNAME) if host and username is None: username = host.connection_param[0].username password = get_row_data(row_data, header_row, HEADER_FIELD_PASSWORD) if host and password is None: password = host.connection_param[0].password enable_password = get_row_data(row_data, header_row, HEADER_FIELD_ENABLE_PASSWORD) if host and enable_password is None: enable_password = host.connection_param[0].enable_password port_number = get_row_data(row_data, header_row, HEADER_FIELD_PORT) if host and port_number is None: port_number = host.connection_param[0].port_number # If no software profile is selected, retain the existing one instead of overwriting it. if host and (software_profile_id is None or software_profile_id <= 0): alternate_software_profile_id = host.software_profile_id else: alternate_software_profile_id = software_profile_id # If no jump host is selected, retain the existing one instead of overwriting it. if host and (jump_host_id is None or jump_host_id <= 0): alternate_jump_host_id = host.connection_param[0].jump_host_id else: alternate_jump_host_id = jump_host_id create_or_update_host(db_session, hostname, alternate_region_id, location, roles, alternate_software_profile_id, connection_type, host_or_ip, username, password, enable_password, port_number, alternate_jump_host_id, created_by, host) except Exception as e: return jsonify({'status': 'Line {} - {} - {}.'.format(row, e.message, row_data)}) row += 1 return jsonify({'status': 'OK'})
def run_conformance_report(profile_name, match_criteria, hostnames): host_not_in_conformance = 0 host_out_dated_inventory = 0 db_session = DBSession() software_profile = get_software_profile(db_session, profile_name) if software_profile is not None: software_profile_packages = software_profile.packages.split(',') conformance_report = ConformanceReport( software_profile=software_profile.name, software_profile_packages=','.join(sorted(software_profile_packages)), match_criteria=match_criteria, hostnames=hostnames, user_id=current_user.id, created_by=current_user.username) for hostname in hostnames.split(','): host = get_host(db_session, hostname) if host: packages_to_match = [software_profile_package.replace('.pie', '') for software_profile_package in software_profile_packages] inventory_job = host.inventory_job[0] comments = '' if inventory_job is not None and inventory_job.last_successful_time is not None: comments = '(' + get_last_successful_inventory_elapsed_time(host) + ')' if inventory_job.status != JobStatus.COMPLETED: comments += ' *' host_out_dated_inventory += 1 host_packages = [] if match_criteria == 'inactive': host_packages = get_host_inactive_packages(hostname) elif match_criteria == 'active': host_packages = get_host_active_packages(hostname) missing_packages = get_missing_packages(host_packages, software_profile_packages) if missing_packages: host_not_in_conformance += 1 conformed = False if len(host_packages) > 0 and len(missing_packages) == 0: conformed = True conformance_report_entry = ConformanceReportEntry( hostname=hostname, platform='Unknown' if host.software_platform is None else host.software_platform, software='Unknown' if host.software_version is None else host.software_version, conformed='Yes' if conformed else 'No', comments=comments, host_packages=','.join(sorted(match_packages(host_packages, packages_to_match))), missing_packages=','.join(sorted(missing_packages))) else: # Flag host not found condition host_out_dated_inventory += 1 host_not_in_conformance += 1 conformance_report_entry = ConformanceReportEntry( hostname=hostname, platform='MISSING', software='MISSING', inventory_status='MISSING', last_successful_retrieval='MISSING', host_packages='MISSING', missing_packages='MISSING') conformance_report.entries.append(conformance_report_entry) conformance_report.host_not_in_conformance = host_not_in_conformance conformance_report.host_out_dated_inventory = host_out_dated_inventory db_session.add(conformance_report) db_session.commit() else: return jsonify({'status': 'Unable to locate the software profile %s' % profile_name}) purge_old_conformance_reports(db_session) return jsonify({'status': 'OK'})
def api_import_hosts(): importable_header = [HEADER_FIELD_HOSTNAME, HEADER_FIELD_REGION, HEADER_FIELD_ROLES, HEADER_FIELD_IP, HEADER_FIELD_USERNAME, HEADER_FIELD_PASSWORD, HEADER_FIELD_CONNECTION, HEADER_FIELD_PORT] region_id = request.form['region'] data_list = request.form['data_list'] db_session = DBSession() selected_region = get_region_by_id(db_session, region_id) if selected_region is None: return jsonify({'status': 'Region is no longer exists in the database.'}) # Check mandatory data fields error = [] reader = csv.reader(data_list.splitlines(), delimiter=',') header_row = next(reader) if HEADER_FIELD_HOSTNAME not in header_row: error.append('"hostname" is missing in the header.') if HEADER_FIELD_IP not in header_row: error.append('"ip" is missing in the header.') if HEADER_FIELD_CONNECTION not in header_row: error.append('"connection" is missing in the header.') for header_field in header_row: if header_field not in importable_header: error.append('"' + header_field + '" is not a correct header field.') if error: return jsonify({'status': ','.join(error)}) # Check if each row has the same number of data fields as the header error = [] data_list = list(reader) row = 2 COLUMN_CONNECTION = get_column_number(header_row, HEADER_FIELD_CONNECTION) COLUMN_REGION = get_column_number(header_row, HEADER_FIELD_REGION) for row_data in data_list: if len(row_data) > 0: if len(row_data) != len(header_row): error.append('line %d has wrong number of data fields.' % row) else: if COLUMN_CONNECTION >= 0: # Validate the connection type data_field = row_data[COLUMN_CONNECTION] if data_field != ConnectionType.TELNET and data_field != ConnectionType.SSH: error.append('line %d has a wrong connection type (should either be "telnet" or "ssh").' % row) if COLUMN_REGION >= 0: # Create a region if necessary data_field = get_acceptable_string(row_data[COLUMN_REGION]) region = get_region(db_session, data_field) if region is None and data_field: try: db_session.add(Region(name=data_field, created_by=current_user.username)) db_session.commit() except Exception: db_session.rollback() error.append('Unable to create region %s.' % data_field) row += 1 if error: return jsonify({'status': ','.join(error)}) # Import the data error = [] im_regions = {} for data in data_list: if len(data) == 0: continue db_host = None im_host = Host() im_host.region_id = selected_region.id im_host.created_by = current_user.username im_host.inventory_job.append(InventoryJob()) im_host.context.append(HostContext()) im_host.connection_param.append(ConnectionParam()) im_host.connection_param[0].username = '' im_host.connection_param[0].password = '' im_host.connection_param[0].port_number = '' for column in range(len(header_row)): header_field = header_row[column] data_field = data[column].strip() if header_field == HEADER_FIELD_HOSTNAME: hostname = get_acceptable_string(data_field) db_host = get_host(db_session, hostname) im_host.hostname = hostname elif header_field == HEADER_FIELD_REGION: region_name = get_acceptable_string(data_field) if region_name in im_regions: im_host.region_id = im_regions[region_name] else: region = get_region(db_session, region_name) if region is not None: im_host.region_id = region.id # Saved for later lookup im_regions[region_name] = region.id elif header_field == HEADER_FIELD_ROLES: im_host.roles = remove_extra_spaces(data_field) elif header_field == HEADER_FIELD_IP: im_host.connection_param[0].host_or_ip = remove_extra_spaces(data_field) elif header_field == HEADER_FIELD_USERNAME: username = get_acceptable_string(data_field) im_host.connection_param[0].username = username elif header_field == HEADER_FIELD_PASSWORD: im_host.connection_param[0].password = data_field elif header_field == HEADER_FIELD_CONNECTION: im_host.connection_param[0].connection_type = data_field elif header_field == HEADER_FIELD_PORT: im_host.connection_param[0].port_number = remove_extra_spaces(data_field) # Import host already exists in the database, just update it if db_host is not None: db_host.created_by = im_host.created_by db_host.region_id = im_host.region_id if HEADER_FIELD_ROLES in header_row: db_host.roles = im_host.roles if HEADER_FIELD_IP in header_row: db_host.connection_param[0].host_or_ip = im_host.connection_param[0].host_or_ip if HEADER_FIELD_USERNAME in header_row: db_host.connection_param[0].username = im_host.connection_param[0].username if HEADER_FIELD_PASSWORD in header_row: db_host.connection_param[0].password = im_host.connection_param[0].password if HEADER_FIELD_CONNECTION in header_row: db_host.connection_param[0].connection_type = im_host.connection_param[0].connection_type if HEADER_FIELD_PORT in header_row: db_host.connection_param[0].port_number = im_host.connection_param[0].port_number else: # Add the import host db_session.add(im_host) if error: return jsonify({'status': error}) else: db_session.commit() return jsonify({'status': 'OK'})
class EONet(object): ''' Object to contain the EONrepr(event['geometries']et events. The event_id, a hashtag, the geometries and the event title are being retained. ''' def __init__(self): self.__session = DBSession().session() self.__events = self.__session.query(DBEONet).all() # Update, for good measure. self.__update_table() # Add the test event. Needed because it's located in Irvine so the # geolocation tweets will match. self.__update_table(TEST_EVENT) def __update_table(self, json_events=None): ''' Makes a dictionary of geometries indexed by hashtags. This supports monitoring the Twitter stream by hashtag and checking the geolocation. The hashtag is formed from each title by removing punctuation, and then combining the first three words. ''' if json_events is None: eonet = json.load(urllib2.urlopen(EONET_URL)) else: eonet = json.loads(json_events) self.events = eonet['events'] print('Requested EONet data: "%s", %d events received.' % (eonet['description'], len(eonet['events'])) ) # Loop through the events saving them to the database. for event in self.events: # Check if the event is already in the database. eonet_id = self.__session.query(DBEONet).filter(DBEONet.eonet_id == event['id']).one_or_none() if eonet_id is None: self.__add_event(event) def __add_event(self, event): '''Add an EONet event to the database.''' # Clean up the title, removing spaces, punctuation and meaningless # words. title = event['title'] exclude = set(string.punctuation) title = ''.join(ch for ch in title if ch not in exclude) wordlist = title.split() hashlist = wordlist[:3] # Assemble the hashtag. hashtag = '#' for keyword in hashlist: hashtag += keyword.capitalize() # The geometries is a list. The python list object should be, # so it's pickled and base64 encoded and then reconstituted when it's # read later. # # The base64 version of the list however isn't meaningful to clients # of the REST API. For that reason, the JSON string representing the # geometries is also saved. geometries = PythonDBObject(event['geometries']) json_geometries = json.dumps(event['geometries']) new = DBEONet(eonet_id = event['id'], title = title, hashtag = hashtag, geometries = geometries.encode(), json_geometries = json_geometries ) self.__session.add(new) self.__session.commit() print('Added "%s": %s (%s).' % (new.eonet_id, new.title, new.hashtag)) def refresh(self): ''' Requests another JSON document of events from EONet and updates the EONet table in the database. ''' self.__update_table() def get_hashtag_from_point(self, point): ''' Determine if the point either lies within any of the polygon boundaries or if it's near a point. If yes, return the first the hashtag of the first matching entry. The OGR Distance() function returns the distance in the units of the coordinate system of the geometries, which in the present case are degree. A degree = 60 nautical miles. ''' results = self.__session.query(DBEONet.hashtag, DBEONet.geometries) for hashtag, b64geometries in results: list_geometries = PythonDBObject(b64geometries).decode() for dict_geometry in list_geometries: json_geometry = json.dumps(dict_geometry) ogr_geometry = ogr.CreateGeometryFromJson(json_geometry) geotype = ogr_geometry.GetGeometryName() if 'POLYGON' == geotype: if point.Within(ogr_geometry): return hashtag # Separation is in degrees, which is converted to miles. if 'POINT' == geotype: deg_separation = point.Distance(ogr_geometry) # The international date line can make points look like # they're almost 360 degrees apart. if deg_separation > 180: deg_separation = 360 - deg_separation sm_separation = degree_to_mile(deg_separation) if sm_separation < POINT_SEPARATION: return hashtag # If nothing's found, return None. return None def point_from_hashtag(self, hashtag): ''' Returns a point given a hashtag. ''' json_geometry = self.__session.query(DBEOnet.json_geometry).filter(DBEOnet.hashtag == hashtag).one_or_none() return json_geometry def records(self): ''' Returns the complete list of events (eonet_id, title), as a list of dictionaries. ''' event_records = [] records = self.__session.query(DBEONet) for record in records: event_records.append(record) return event_records def hashtags(self): '''Returns the complete list of the hashtags representing the events.''' hashtags = [] results = self.__session.query(DBEONet.hashtag) for result in results: hashtag = result[0].encode('utf8', 'replace') hashtags.append(hashtag) results = self.__session.query(DBEONet.eonet_id) for result in results: hashtag = '#'+result[0].encode('utf8', 'replace') hashtags.append(hashtag) return hashtags
class SQLDump(TweetConsumer): ''' Saves the parts of the tweet that the REST API requires to a database. ''' def __init__(self): self.__session = DBSession().session() def _save(self, tweet, hashtag=None): # Obtain the event from the database. event = self.__session.query(DBEONet).filter(DBEONet.hashtag == hashtag).one_or_none() # No need to continue if the tweet is already in the database. db_tweet = self.__session.query(DBTweets).filter(DBTweets.tweet_id == tweet.id_str).one_or_none() if db_tweet is not None: return # the data fields to save. name = tweet.user.name create_date = tweet.created_at place = tweet.place.name if tweet.place else None text = tweet.text profile_pic = tweet.user.profile_image_url screen_name = tweet.user.screen_name try: tweet.extended_entities except AttributeError: media_url = None else: media_url = tweet.extended_entities['media'][0]['media_url'] if tweet.extended_entities['media'] and tweet.extended_entities['media'][0]['type'] == 'photo' else None # The coordinates of the tweet are either the specific long, lat # provided in the tweet, or the centroid of the place's bounding # box. # # The coordinates is a list that must be saved as a string. However # the list needs to be reconstituted when it's read. The clients # consuming the REST API need the JSON version of the coordinates. if tweet.coordinates is not None: coordinates = PythonDBObject(tweet.coordinates) json_coordinates = json.dumps(tweet.coordinates) # Second choice is the centroid of the place elif tweet.place is not None: center = place_centroid(tweet) # Build the dictionary. l_coords = [center.GetX(), center.GetY()] d_coords = {u'type': 'Point', u'coordinates': l_coords} coordinates = PythonDBObject(d_coords) json_coordinates = json.dumps(d_coords) # Use the centroid of the event if we have that. elif hashtag is not None: e_json = event.json_geometries f_json = str(e_json) g_json = f_json[1:-1] ogr_geometry = ogr.CreateGeometryFromJson(g_json) geotype = ogr_geometry.GetGeometryName if 'Point' == geotype: coordinates = PythonDBObject(g_json) json_coordinates =g_json else: center = ogr_geometry.Centroid() # Build the dictionary. l_coords = [center.GetX(), center.GetY()] d_coords = {u'type': 'Point', u'coordinates': l_coords} coordinates = PythonDBObject(d_coords) json_coordinates = json.dumps(d_coords) # Commit to the database. tweet_record = DBTweets(tweet_id=tweet.id_str, eonet_id=event.eonet_id, name=name, createDate=create_date, place=place, msg=text, coordinates=coordinates.encode(), hashtag=hashtag, media_url=media_url, screen_name=screen_name, profile_pic=profile_pic, json_coordinates=json_coordinates) self.__session.add(tweet_record) self.__session.commit() # Write the same to standard out to monitor the progress. The terminal # requires the text to be encoded as utf8. text = text.encode('utf8', 'replace') name = name.encode('utf8', 'replace') place = place.encode('utf8', 'replace') if place else None media_url = media_url.encode('utf8', 'replace') if media_url else None print('Written to the database: [Name: %s][Place: %s][At: %s] %s %s.' % (name, place, create_date, text, media_url))
def init_system_option(): db_session = DBSession() if db_session.query(SystemOption).count() == 0: db_session.add(SystemOption()) db_session.commit()