def save_to_mongo(request_json):
try:
Database.insert(GraphConstants.COLLECTION, request_json)
except:
GraphErrors.GraphAlreadyExistsError(
"The graph you tried to create with name {} already exists. Please pick a different 'unique_name'.".format(
request_json['unique_name']))
def updatedb(self):
try:
db = Database()
smsen = []
smsen.append(self.smsdict)
smsgwglobals.wislogger.debug("WATCHDOG: " +
"UPDATING " + str(self.smsdict["smsid"]) + " " + str(self.smsdict["status"]))
db.update_sms(smsen)
except error.DatabaseError as e:
smsgwglobals.wislogger.debug(e.message)
def delete_months(group, months_to_keep):
# Find data in the database
# Those timestamps older than `months`, delete them.
if months_to_keep < 1:
return
else:
Database.remove(DataConstants.COLLECTION,
{"group": group,
"date":
{"$lte": datetime.datetime.utcnow() - datetime.timedelta(days=months_to_keep * 31)}
})
def run(self):
# load the configuration
# Create default root user
db = Database()
abspath = path.abspath(path.join(path.dirname(__file__), path.pardir))
configfile = abspath + '/conf/smsgw.conf'
cfg = SmsConfig(configfile)
wisglobals.wisid = cfg.getvalue('wisid', 'nowisid', 'wis')
wisglobals.wisipaddress = cfg.getvalue('ipaddress', '127.0.0.1', 'wis')
wisglobals.wisport = cfg.getvalue('port', '7777', 'wis')
wisglobals.cleanupseconds = cfg.getvalue('cleanupseconds', '86400', 'wis')
wisglobals.ldapserver = cfg.getvalue('ldapserver', None, 'wis')
wisglobals.ldapbasedn = cfg.getvalue('ldapbasedn', None, 'wis')
wisglobals.ldapenabled = cfg.getvalue('ldapenabled', None, 'wis')
ldapusers = cfg.getvalue('ldapusers', '[]', 'wis')
wisglobals.ldapusers = json.loads(ldapusers)
wisglobals.ldapusers = [item.lower() for item in wisglobals.ldapusers]
smsgwglobals.wislogger.debug("WIS:" + str(wisglobals.ldapusers))
password = cfg.getvalue('password', '20778ba41791cdc8ac54b4f1dab8cf7602a81f256cbeb9e782263e8bb00e01794d47651351e5873f9ac82868ede75aa6719160e624f02bba4df1f94324025058', 'wis')
salt = cfg.getvalue('salt', 'changeme', 'wis')
# write the default user on startup
db.write_users('root', password, salt)
# check if ssl is enabled
wisglobals.sslenabled = cfg.getvalue('sslenabled', None, 'wis')
wisglobals.sslcertificate = cfg.getvalue('sslcertificate', None, 'wis')
wisglobals.sslprivatekey = cfg.getvalue('sslprivatekey', None, 'wis')
wisglobals.sslcertificatechain = cfg.getvalue('sslcertificatechain', None, 'wis')
smsgwglobals.wislogger.debug("WIS: SSL " + str(wisglobals.sslenabled))
if wisglobals.sslenabled is not None and 'true' in wisglobals.sslenabled.lower():
smsgwglobals.wislogger.debug("WIS: STARTING SSL")
cherrypy.config.update({'server.ssl_module':
'builtin'})
cherrypy.config.update({'server.ssl_certificate':
wisglobals.sslcertificate})
cherrypy.config.update({'server.ssl_private_key':
wisglobals.sslprivatekey})
if wisglobals.sslcertificatechain is not None:
cherrypy.config.update({'server.ssl_certificate_chain':
wisglobals.sslcertificatechain})
cherrypy.config.update({'server.socket_host':
wisglobals.wisipaddress})
cherrypy.config.update({'server.socket_port':
int(wisglobals.wisport)})
cherrypy.quickstart(Root(), '/smsgateway',
'wis-web.conf')
def writetodb(self):
try:
db = Database()
db.insert_sms(self.smsdict["modemid"],
self.smsdict["targetnr"],
self.smsdict["content"],
self.smsdict["priority"],
self.smsdict["appid"],
self.smsdict["sourceip"],
self.smsdict["xforwardedfor"],
self.smsdict["smsintime"],
self.smsdict["status"],
self.smsdict["statustime"])
except error.DatabaseError as e:
smsgwglobals.wislogger.debug(e.message)
def from_name(cls, name, **kwargs):
data = Database.find_one(GraphConstants.COLLECTION, {'unique_name': name})
if not data:
raise GraphErrors.GraphNotFoundError("The graph you requested '{}' was not found.".format(name))
data.update(kwargs)
graph = cls.from_json(data)
return graph.run(data)
def checkpassword(username, password):
try:
db = Database()
erg = db.read_users(username)
if erg is None or len(erg) == 0:
raise error.UserNotFoundError()
retpasswordhash = erg[0]["password"]
retpasswordsalt = erg[0]["salt"]
givpasswordhash = hashlib.sha512((password +
retpasswordsalt)
.encode('utf-8')).hexdigest()
if retpasswordhash == givpasswordhash:
return True
else:
return False
except error.DatabaseError as e:
smsgwglobals.wislogger.debug(e.message)
def login_valid_user(email, password):
"""
:param email: user's email
:param password: user's password
:return: True if valid, False otherwise
"""
userData = Database.find_one("users", {'email': email})
if userData is None:
raise UserNotExistsError("this user does not existed!")
return False
if not Utils.check_hashed_password(password, userData['password']):
raise UserIncorrectPasswordError("Incorrect password!")
return False
return True
def is_login_valid(cls, email, password):
"""
This method verifies that an e-mail/password combo (as sent by the site forms) is valid or not.
Checks that the e-mail exists, and that the password associated to that e-mail is correct.
:param email: The user's email
:param password: A sha512 hashed password
:return: True if valid, False otherwise
"""
user_data = Database.find_one(UserConstants.COLLECTION, {"email": email}) # Password in sha512 -> pbkdf2_sha512
if user_data is None:
# Tell the user that their e-mail doesn't exist
raise UserErrors.UserNotExistsError("Your user does not exist.")
if not Utils.check_hashed_password(password, user_data['password']):
# Tell the user that their password is wrong
raise UserErrors.IncorrectPasswordError("Your password was wrong.")
return cls(**user_data)
def register_user(email, password):
"""
This method registers a user using e-mail and password.
The password already come with sha-512 hash algorithm.
:param email: user's email (might be invalid)
:param password: sha512-hashed password
:return: True if registered successfully, of False otherwise
"""
userData = Database.find_one("users", {'email': email})
if userData is not None:
raise UserExistsError("user already existed!")
#if email is invalid, then what?
if not Utils.email_is_valid(email):
raise UserEmailInvalid("invalid email!")
#hash password, create new object, then insert to db
User(email, Utils.hash_password(password)).save_to_db()
return True
def register_user(email, password):
"""
This method registers a user using e-mail and password.
The password already comes hashed as sha-512.
:param email: user's e-mail (might be invalid)
:param password: sha512-hashed password
:return: True if registered successfully, or False otherwise (exceptions can also be raised)
"""
user_data = Database.find_one(UserConstants.COLLECTION, {"email": email})
if user_data is not None:
raise UserErrors.UserAlreadyRegisteredError("The e-mail you used to register already exists.")
if not Utils.email_is_valid(email):
raise UserErrors.InvalidEmailError("The e-mail does not have the right format.")
user = User(email, Utils.hash_password(password))
user.save_to_db()
return user
def _remove_fr_mongo(email):
Database.remove(collection="users", data={"email": email})
def from_mongo(cls, id):
blog_data = Database.find_one(collection='blogs', query={'_id': id})
return cls(**blog_data)
def get_all(cls):
users = Database.find(collection="users", query=({}))
return [cls(**user) for user in users]
def from_mongo(cls, id):
post_data = Database.find_one(collection="posts", query={"_id": id})
return cls(**post_data)
def find_all_ganti_password(cls):
return [
cls(**data) for data in Database.find(cls.collection, {})
if cls(**data).ganti_password != ''
]
def main(self):
if self.brute is None:
self.brute = config.enable_brute_module
if self.dns is None:
self.dns = config.enable_dns_resolve
if self.req is None:
self.req = config.enable_http_request
old_table = self.domain + '_last'
new_table = self.domain + '_now'
collect = Collect(self.domain, export=False)
collect.run()
if self.brute:
# 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
brute = AIOBrute(self.domain, export=False)
brute.run()
db = Database()
db.copy_table(self.domain, self.domain + '_ori')
db.remove_invalid(self.domain)
db.deduplicate_subdomain(self.domain)
old_data = []
# 非第一次收集子域的情况时数据库预处理
if db.exist_table(new_table):
db.drop_table(old_table) # 如果存在上次收集结果表就先删除
db.rename_table(new_table, old_table) # 新表重命名为旧表
old_data = db.get_data(old_table).as_dict()
# 不解析子域直接导出结果
if not self.dns:
# 数据库导出
self.valid = None
dbexport.export(self.domain,
valid=self.valid,
format=self.format,
show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
db.close()
return
self.data = db.get_data(self.domain).as_dict()
# 标记新发现子域
self.data = utils.mark_subdomain(old_data, self.data)
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
# 解析子域
task = resolve.bulk_query_a(self.data)
self.data = loop.run_until_complete(task)
# 保存解析结果
resolve_table = self.domain + '_res'
db.drop_table(resolve_table)
db.create_table(resolve_table)
db.save_db(resolve_table, self.data, 'resolve')
# 不请求子域直接导出结果
if not self.req:
# 数据库导出
self.valid = None
dbexport.export(self.domain,
valid=self.valid,
format=self.format,
show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
db.close()
return
# 请求子域
task = request.bulk_get_request(self.data, self.port)
self.data = loop.run_until_complete(task)
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25))
logger.log('INFOR', f'经验证{self.domain}有效子域{len(self.data)}个')
# 保存请求结果
db.clear_table(self.domain)
db.save_db(self.domain, self.data, 'request')
# 数据库导出
dbexport.export(self.domain,
valid=self.valid,
format=self.format,
show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
db.close()
# 子域接管检查
if self.takeover:
subdomains = set(map(lambda x: x.get('subdomain'), self.data))
takeover = Takeover(subdomains)
takeover.run()
def get_by_email(cls, email):
data = Database.find_one("users", {"email": email})
if data is not None:
return cls(**data)
def api(self, arg, **params):
cl = cherrypy.request.headers['Content-Length']
rawbody = cherrypy.request.body.read(int(cl))
smsgwglobals.wislogger.debug(rawbody)
plaintext = GlobalHelper.decodeAES(rawbody)
smsgwglobals.wislogger.debug(plaintext)
data = json.loads(plaintext)
if arg == "watchdog":
if data["run"] == "True":
self.triggerwatchdog()
else:
cherrypy.response.status = 400
if arg == "heartbeat":
if "routingid" in data:
smsgwglobals.wislogger.debug(data["routingid"])
try:
count = wisglobals.rdb.raise_heartbeat(data["routingid"])
if count == 0:
smsgwglobals.wislogger.debug("COUNT: " + str(count))
cherrypy.response.status = 400
except error.DatabaseError:
cherrypy.response.status = 400
else:
cherrypy.response.status = 400
if arg == "receiverouting":
try:
wisglobals.rdb.merge_routing(data)
except error.DatabaseError as e:
smsgwglobals.wislogger.debug(e.message)
if arg == "requestrouting":
if data["get"] != "peers":
cherrypy.response.status = 400
return
smsgwglobals.wislogger.debug("Sending routing table to you")
try:
erg = wisglobals.rdb.read_routing()
jerg = json.dumps(erg)
data = GlobalHelper.encodeAES(jerg)
return data
except error.DatabaseError as e:
smsgwglobals.wislogger.debug(e.message)
if arg == "managemodem":
try:
if data["action"] == "register":
smsgwglobals.wislogger.debug("managemodem register")
smsgwglobals.wislogger.debug(wisglobals.wisid)
# add wisid to data object
data["wisid"] = wisglobals.wisid
# store date in routing table
wisglobals.rdb.write_routing(data)
# call receiverouting to distribute routing
Helper.receiverouting()
elif data["action"] == "unregister":
smsgwglobals.wislogger.debug("managemodem unregister")
wisglobals.rdb.change_obsolete(data["routingid"], 14)
Helper.receiverouting()
else:
return False
except error.DatabaseError as e:
smsgwglobals.wislogger.debug(e.message)
if arg == "deligatesms":
if "sms" in data:
smsgwglobals.wislogger.debug(data["sms"])
try:
sms = Smstransfer(**data["sms"])
sms.smsdict["status"] = 1
sms.writetodb()
self.triggerwatchdog()
except error.DatabaseError:
cherrypy.response.status = 400
else:
cherrypy.response.status = 400
if arg == "router":
if data["action"] == "status":
smsgwglobals.wislogger.debug("API: " + data["action"])
if wisglobals.routerThread is None:
cherrypy.response.status = 200
data = GlobalHelper.encodeAES('{"ROUTER":"noobject"}')
return data
if wisglobals.routerThread.isAlive():
cherrypy.response.status = 200
data = GlobalHelper.encodeAES('{"ROUTER":"alive"}')
return data
else:
cherrypy.response.status = 200
data = GlobalHelper.encodeAES('{"ROUTER":"dead"}')
return data
if arg == "getsms":
if data["get"] != "sms":
cherrypy.response.status = 400
return
if "date" in data:
date = data["date"]
smsgwglobals.wislogger.debug("API: " + date)
else:
date = None
smsgwglobals.wislogger.debug("Sending SMS Table")
smsgwglobals.wislogger.debug("Sending SMS Table date: " + str(date))
try:
db = Database()
erg = db.read_sms_date(date=date)
jerg = json.dumps(erg)
data = GlobalHelper.encodeAES(jerg)
return data
except error.DatabaseError as e:
smsgwglobals.wislogger.debug(e.message)
def from_all():
return [movie for movie in Database.find(collection='movies')]
def mark(self):
"""
Mark the new discovered subdomain
:return: marked data
:rtype: list
"""
db = Database()
old_data = list()
now_data = db.get_data(self.domain).as_dict()
# Database pre-processing when it is not the first time to collect this subdomain
if db.exist_table(self.new_table):
# If there is the last collection result table, delete it first
db.drop_table(self.old_table)
# Rename the new table to the old table
db.rename_table(self.new_table, self.old_table)
old_data = db.get_data(self.old_table).as_dict()
db.close()
marked_data = utils.mark_subdomain(old_data, now_data)
return marked_data
def from_mongo(cls, file):
movie_data = Database.find_one(collection='movies',
query={'file': file})
return movie_data
def find_by_user_email(cls, user_email):
return [
cls(**elem) for elem in Database.find(AlertConstants.COLLECTION,
{"user_email": user_email})
]
def save_to_mongo(self):
Database.update(AlertConstants.COLLECTION, {"_id": self._id},
self.json())
def find_one_warga_from_db(self):
return Database.find_one(Warga.collection, {
"blok_ppn": self.blok_ppn,
"no_ppn": self.no_ppn
})
def get_by_id(cls, _id):
data = Database.find_one("users", {"_id": _id})
if data is not None:
return cls(**data)
def get_by_id(cls, _id):
data = Database.find_one("users", {"_id": _id})
if data is not None:
return cls(**data)
return None
def get_balance(username):
data = Database.find_one("users", {"username": username})
return data["balance"]
def save_one_to_db(self):
Database.insert_one(Warga.collection, self.json())
def save_to_mongo(self):
Database.insert("users", self.json())
def delete(self):
Database.remove(AlertConstants.COLLECTION, {"_id": self._id})
def save_to_mongo(self):
Database.insert(collection='students', data=self.json())
def deactivate(self):
Database.update(collection="users",
query={"email": self.email},
new_val={"$set": {
"active": False
}})
def from_mongo_by_id(cls, _id):
# Post.from_mongo('123')
student_data = Database.find_one(collection='students',
query={'_id': _id})
return cls(**student_data)
def get_by_email(cls, email):
data = Database.find_one(collection="users", query={"email": email})
if data is not None:
return cls(**data)
def from_mongo_by_email(cls, email):
data = Database.find_one(collection='students', query={'email': email})
if data is not None:
return cls(**data)
return None
def from_blog(id):
return [post for post in Database.find(collection="posts", query={"blog_id": id})]
def all_from_mongo(cls):
data = Database.find(collection='students', query={})
return cls(**data)
def init_db():
Database.initialize()
def save_to_db(self):
Database.insert(UserConstants.COLLECTION, self.json())
def find_by_author_id(cls, author_id):
blogs = Database.find(collection='blogs',
query={'author_id': author_id})
return [cls(**blog) for blog in blogs]
def api_key_exists(api_key):
return Database.find_one(UserConstants.COLLECTION, {'api_key': api_key}) is not None
def get_by_email(cls, email):
data = Database.find_one("users", {"email": email})
if data is not None:
return cls(**data)
def find_by_group(cls, group):
return cls(**Database.find_one(AggregatorConstants.COLLECTION, {"group": group}))
def save_to_mongo(self):
Database.insert("users", self.json())
def from_mongo(cls, id):
post_data = Database.find_one(collection='posts', query={'_id': id})
return cls(**post_data)
def run(self):
# load the configuration
# Create default root user
db = Database()
abspath = path.abspath(path.join(path.dirname(__file__), path.pardir))
# store the abspath in globals for easier handling
wisglobals.smsgatewayabspath = abspath
configfile = abspath + '/conf/smsgw.conf'
cfg = SmsConfig(configfile)
readme = open(abspath + '/README.md', 'r')
readmecontent = readme.read()
version = re.compile(r"(?<=## Version)(.*v.\..*)", re.S).findall(readmecontent)
if version:
wisglobals.version = version[0].strip('\n')
smsgwglobals.wislogger.debug("WIS: Version: " + str(wisglobals.version))
wisglobals.wisid = cfg.getvalue('wisid', 'nowisid', 'wis')
wisglobals.wisipaddress = cfg.getvalue('ipaddress', '127.0.0.1', 'wis')
wisglobals.wisport = cfg.getvalue('port', '7777', 'wis')
wisglobals.cleanupseconds = cfg.getvalue('cleanupseconds', '86400', 'wis')
wisglobals.validusernameregex = cfg.getvalue('validusernameregex', '([^a-zA-Z0-9])', 'wis')
wisglobals.validusernamelength = cfg.getvalue('validusernamelength', 30, 'wis')
wisglobals.ldapserver = cfg.getvalue('ldapserver', None, 'wis')
wisglobals.ldapbasedn = cfg.getvalue('ldapbasedn', None, 'wis')
wisglobals.ldapenabled = cfg.getvalue('ldapenabled', None, 'wis')
ldapusers = cfg.getvalue('ldapusers', '[]', 'wis')
wisglobals.ldapusers = json.loads(ldapusers)
wisglobals.ldapusers = [item.lower() for item in wisglobals.ldapusers]
smsgwglobals.wislogger.debug("WIS:" + str(wisglobals.ldapusers))
password = cfg.getvalue('password', '20778ba41791cdc8ac54b4f1dab8cf7602a81f256cbeb9e782263e8bb00e01794d47651351e5873f9ac82868ede75aa6719160e624f02bba4df1f94324025058', 'wis')
salt = cfg.getvalue('salt', 'changeme', 'wis')
# write the default user on startup
db.write_users('root', password, salt)
# read pissendtimeout
wisglobals.pissendtimeout = int(cfg.getvalue('pissendtimeout', '20', 'wis'))
# check if ssl is enabled
wisglobals.sslenabled = cfg.getvalue('sslenabled', None, 'wis')
wisglobals.sslcertificate = cfg.getvalue('sslcertificate', None, 'wis')
wisglobals.sslprivatekey = cfg.getvalue('sslprivatekey', None, 'wis')
wisglobals.sslcertificatechain = cfg.getvalue('sslcertificatechain', None, 'wis')
smsgwglobals.wislogger.debug("WIS: SSL " + str(wisglobals.sslenabled))
if wisglobals.sslenabled is not None and 'true' in wisglobals.sslenabled.lower():
smsgwglobals.wislogger.debug("WIS: STARTING SSL")
cherrypy.config.update({'server.ssl_module':
'builtin'})
cherrypy.config.update({'server.ssl_certificate':
wisglobals.sslcertificate})
cherrypy.config.update({'server.ssl_private_key':
wisglobals.sslprivatekey})
if wisglobals.sslcertificatechain is not None:
cherrypy.config.update({'server.ssl_certificate_chain':
wisglobals.sslcertificatechain})
cherrypy.config.update({'server.socket_host':
wisglobals.wisipaddress})
cherrypy.config.update({'server.socket_port':
int(wisglobals.wisport)})
cherrypy.tree.mount(StatsLogstash(), '/smsgateway/api/stats/logstash', {'/': {'request.dispatch': cherrypy.dispatch.MethodDispatcher()}})
cherrypy.quickstart(Root(), '/smsgateway',
'wis-web.conf')
from common.database import Database
db = Database.initialize()
tweets = Database.find(collection='twitter_search', query={})
lis = []
neg = 0.0
n = 0.0
net = 0.0
pos = 0.0
p = 0.0
cout = 0
for tweet in tweets:
# Create a list with all the terms
blob = TextBlob(tweet["text"])
cout += 1
lis.append(blob.sentiment.polarity)
# print blob.sentiment.subjectivity
# print (os.listdir(tweet["text"]))
if blob.sentiment.polarity < 0:
sentiment = "negative"
neg += blob.sentiment.polarity
n += 1
elif blob.sentiment.polarity == 0:
sentiment = "neutral"
net += 1
else:
sentiment = "positive"
pos += blob.sentiment.polarity
p += 1
def initialize_database(): Database.initialize()
def init_db():
Database.initialize()
def update(self, attrib, val):
Database.update(collection="books",
query={"_id": self._id},
new_val={"$set": {
attrib: val
}})
def find_by_id(cls, alert_id):
return cls(
**Database.find_one(AlertConstants.COLLECTION, {"_id": alert_id}))
def find_by_id(cls, id):
return cls(**Database.find_one(UserConstants.COLLECTION, {'_id': id}))
def get_all(cls):
books = Database.find(collection="books", query=({}))
return [cls(**book) for book in books]
def save_to_mongo(self):
Database.insert(AggregatorConstants.COLLECTION, self.json())
def get_by_title(cls, name):
data = Database.find_one(collection="books", query={"title": title})
if data is not None:
return cls(**data)
def save_to_mongo(self):
Database.insert(collection='posts',
data=self.json())
def get_by_id(cls, _id):
data = Database.find_one(collection="books", query={"_id": _id})
if data is not None:
return cls(**data)
def from_blog(id):
return [post for post in Database.find(collection='posts', query={'blog_id': id})]
def save_to_mongo(self):
Database.insert(collection="books", data=self.json())