class JsonDatabase(object):
def __init__(self, database_path):
self.database = Database(database_path)
def find(self, key):
return self.database.data(key=key)
def insert(self, key, values):
self.database[key] = values
return values
def update(self, key, **kwargs):
values = self.database.data(key=key)
for parameter in kwargs:
values[parameter] = kwargs[parameter]
self.database[key] = values
return values
def keys(self):
return list(self.database[0].keys())
def get_random(self):
statement = random.choice(self.keys())
return {statement: self.find(statement)}
def get_closest_statement(text, log_directory):
"""
Takes a statement from a conversation.
Returns a the closest matching statement in the database.
"""
import os
from jsondb.db import Database
from fuzzywuzzy import fuzz
import random
closest_statement = None
closest_ratio = 0
database = Database(log_directory)
data = database.data()
# Return immediately if an exact match exists
if text in data:
return {text: data[text]}
for statement in data:
ratio = fuzz.ratio(statement, text)
if ratio > closest_ratio:
closest_ratio = ratio
closest_statement = statement
# If the ratios are the same, pick the one to keep at random
elif ratio == closest_ratio and closest_ratio != 0:
# Use a random boolean to determine which statement to keep
if bool(random.getrandbits(1)):
closest_statement = statement
return {closest_statement: data[closest_statement]}
def add_article(feed, parsed, entry, guid, message):
"""Add article to the database.
A post processing hook for rss2email, run on every new entry/article.
"""
path = join(HERE, 'inbox', 'digest.json')
db = Database(path)
key = random_string()
data = {
'content':
entry.get('summary', 'no summary'),
'title':
entry.get('title', 'no title'),
'url':
entry.get('link', 'no link'),
'author':
entry.get('author', ''),
'blog':
parsed.get('feed', {}).get('title', ''),
'date_published':
time.strftime('%Y-%m-%dT%H:%M:%S%z',
entry.get('updated_parsed', time.localtime())),
'date_added':
time.strftime('%Y-%m-%dT%H:%M:%S%z', time.localtime()),
}
db.data(key=key, value=data)
return message
def get_closest_statement(text, log_directory):
"""
Takes a statement from a conversation.
Returns a the closest matching statement in the database.
"""
import os
from jsondb.db import Database
from fuzzywuzzy import fuzz
import random
closest_statement = None
closest_ratio = 0
database = Database(log_directory)
data = database.data()
# Return immediately if an exact match exists
if text in data:
return {text: data[text]}
for statement in data:
ratio = fuzz.ratio(statement, text)
if ratio > closest_ratio:
closest_ratio = ratio
closest_statement = statement
# If the ratios are the same, pick the one to keep at random
elif ratio == closest_ratio and closest_ratio != 0:
# Use a random boolean to determine which statement to keep
if bool(random.getrandbits(1)):
closest_statement = statement
return {closest_statement: data[closest_statement]}
class JsonDatabaseAdapter:
def __init__(self, database_path):
self.database = Database(database_path)
def find(self, key):
return self.database.data(key=key)
def insert(self, key, values):
self.database[key] = values
return values
def update(self, key, **kwargs):
values = self.database.data(key=key)
for parameter in kwargs:
values[parameter] = kwargs[parameter]
self.database[key] = values
return values
def keys(self):
# The return value has to be cast as a list for Python 3 compatibility
return list(self.database[0].keys())
def get_random(self):
"""
Returns a random statement from the database
"""
statement = random.choice(self.keys())
return {statement: self.find(statement)}
class InMemoryTests(TestCase):
def setUp(self):
self.database = Database(None)
def test_nonexistant_database_is_invalid(self):
self.database.data(key="memory", value="yes")
self.assertEqual(self.database.data(key="memory"), "yes")
class InMemoryTests(TestCase):
def setUp(self):
self.database = Database(None)
def test_nonexistant_database_is_invalid(self):
self.database.data(key="memory", value="yes")
self.assertEqual(self.database.data(key="memory"), "yes")
def export_for_training(self, file_path='./export.json'):
"""
Create a file from the database that can be used to
train other chat bots.
"""
from jsondb.db import Database
database = Database(file_path)
export = {'export': self._generate_export_data()}
database.data(dictionary=export)
def export_for_training(self, file_path='./export.json'):
"""
Create a file from the database that can be used to
train other chat bots.
"""
from jsondb.db import Database
database = Database(file_path)
export = {'export': self._generate_export_data()}
database.data(dictionary=export)
def __init__(self, db_path):
self.db = Database(db_path)
self.db.data(dictionary={
"meta": "",
"platform": [],
"asset": [],
"hdwseed": []
})
self.platform = keystoreManager(KeyType.PLATFORM, self.db)
self.asset = keystoreManager(KeyType.ASSET, self.db)
def _mark_entries_as_digested(db_path, entries):
db = Database(db_path)
for key, value in entries.items():
tags = value.setdefault('tags', [])
if tags is None:
tags = []
if 'digest' not in tags:
tags.append('digest')
db.data(key=key, value=value)
def __init__(self, **kwargs):
super(JsonFileStorageAdapter, self).__init__(**kwargs)
if not kwargs.get('silence_performance_warning', False):
warnings.warn(
'The JsonFileStorageAdapter is not recommended for production application environments.',
self.UnsuitableForProductionWarning)
database_path = self.kwargs.get('database', 'database.db')
self.database = Database(database_path)
self.adapter_supports_queries = False
class JsonDatabaseAdapter(DatabaseAdapter):
def __init__(self, database_path):
self.database = Database(database_path)
def _keys(self):
# The value has to be cast as a list for Python 3 compatibility
return list(self.database[0].keys())
def count(self):
return len(self._keys())
def find(self, key):
return self.database.data(key=key)
def insert(self, key, values):
self.database[key] = values
return values
def update(self, key, **kwargs):
values = self.database.data(key=key)
# Create the statement if it doesn't exist in the database
if not values:
self.database[key] = {}
values = {}
for parameter in kwargs:
values[parameter] = kwargs.get(parameter)
self.database[key] = values
return values
def get_random(self):
from random import choice
if self.count() < 1:
raise EmptyDatabaseException()
statement = choice(self._keys())
return {statement: self.find(statement)}
def drop(self):
"""
Remove the json file database completely
"""
import os
os.remove(self.database.path)
def __init__(self, log_key):
"""
Takes a string parameter that will be used as a
key to store logged data under in the database.
"""
self.database = Database("settings.db")
self.start_time = 0
self.log_key = log_key
def get_response_times(self, analytics_key):
from jsondb.db import Database
database = Database("settings.db")
if analytics_key not in database:
return []
return database[analytics_key]
def _get_entries(db_path):
db = Database(db_path)
data = db.data()
last_digest_timestamp = data.pop('last-digest-timestamp', None)
if last_digest_timestamp is not None:
data = {
guid: entry for guid, entry in data.items()
if entry['date_added'] > last_digest_timestamp
}
data = {
guid: entry for guid, entry in data.items()
if 'read' not in (entry.get('tags') or [])
}
return data
def get_api_response_times(self):
from jsondb.db import Database
database = Database("settings.db")
if not "api_response_time" in database:
return []
return database["api_response_time"]
def __init__(self, name="bot", logging=True):
from jsondb.db import Database
self.name = name
self.log = logging
self.last_statement = None
self.database = Database("database.db")
class JsonDatabaseAdapter(DatabaseAdapter):
def __init__(self, database_path):
self.database = Database(database_path)
def find(self, key):
return self.database.data(key=key)
def insert(self, key, values):
self.database[key] = values
return values
def update(self, key, **kwargs):
values = self.database.data(key=key)
# Create the statement if it doesn't exist in the database
if not values:
self.database[key] = {}
values = {}
for parameter in kwargs:
values[parameter] = kwargs.get(parameter)
self.database[key] = values
return values
def keys(self):
# The value has to be cast as a list for Python 3 compatibility
return list(self.database[0].keys())
def get_random(self):
"""
Returns a random statement from the database
"""
from random import choice
statement = choice(self.keys())
return {statement: self.find(statement)}
def setUp(self):
import json
content = {
"url": "http://sky.net",
"ip": "http://10.0.1.337",
}
with open("test.db", "w+") as test:
json.dump(content, test)
self.database = Database("test.db")
def exist(self, **kwargs):
db_path = kwargs.get("db_path", "keystore.db")
db = Database(db_path)
meta = db["meta"]
platform = db["platform"]
asset = db["asset"]
hdwseed = db["hdwseed"]
return ((meta is not None and meta != "")
or (platform is not None and len(platform) != 0)
or (asset is not None and len(asset) != 0)
or (hdwseed is not None and len(hdwseed) != 0))
class CCkey:
def __init__(self, db_path):
self.db = Database(db_path)
self.db.data(dictionary={
"meta": "",
"platform": [],
"asset": [],
"hdwseed": []
})
self.platform = keystoreManager(KeyType.PLATFORM, self.db)
self.asset = keystoreManager(KeyType.ASSET, self.db)
@staticmethod
def create(**kwargs):
db_path = kwargs.get("db_path", "keystore.db")
return CCkey(db_path)
def exist(self, **kwargs):
db_path = kwargs.get("db_path", "keystore.db")
db = Database(db_path)
meta = db["meta"]
platform = db["platform"]
asset = db["asset"]
hdwseed = db["hdwseed"]
return ((meta is not None and meta != "")
or (platform is not None and len(platform) != 0)
or (asset is not None and len(asset) != 0)
or (hdwseed is not None and len(hdwseed) != 0))
def get_meta(self):
return self.db["meta"]
def set_meta(self, meta: str):
self.db["meta"] = meta
return
def add_article(feed, parsed, entry, guid, message):
"""Add article to the database.
A post processing hook for rss2email, run on every new entry/article.
"""
path = join(HERE, 'inbox', 'digest.json')
db = Database(path)
key = random_string()
data = {
'content': entry.get('summary', 'no summary'),
'title': entry.get('title', 'no title'),
'url': entry.get('link', 'no link'),
'author': entry.get('author', ''),
'blog': parsed.get('feed', {}).get('title', ''),
'date_published': time.strftime('%Y-%m-%dT%H:%M:%S%z',
entry.get('updated_parsed', time.localtime())),
'date_added': time.strftime('%Y-%m-%dT%H:%M:%S%z', time.localtime()),
}
db.data(key=key, value=data)
return message
def engram(text, log_directory):
"""
Returns the statement after the closest matchng statement in
the conversation.
"""
import os
from jsondb.db import Database
import random
matching_responces = []
database = Database(log_directory).data()
# If no text was passed in, then return a random statement.
if not text or not text.strip():
selection = random.choice(list(database.keys()))
return {selection: database[selection]}
closest_statement = get_closest_statement(text, log_directory)
closest_statement_key = list(closest_statement.keys())[0]
for statement in database:
if "in_response_to" in database[statement]:
in_response_to = database[statement]["in_response_to"]
# check if our closest statement is in this list
if closest_statement_key in in_response_to:
matching_responces.append(statement)
# If no matching responses are found: return something random
if not matching_responces:
selection = random.choice(list(database.keys()))
return {selection: database[selection]}
# Choose from the selection of matching responces
selection = random.choice(matching_responces)
return {selection: database[selection]}
def engram(text, log_directory):
"""
Returns the statement after the closest matchng statement in
the conversation.
"""
import os
from jsondb.db import Database
import random
matching_responces = []
database = Database(log_directory).data()
# If no text was passed in, then return a random statement.
if not text or not text.strip():
selection = random.choice(list(database.keys()))
return {selection: database[selection]}
closest_statement = get_closest_statement(text, log_directory)
closest_statement_key = list(closest_statement.keys())[0]
for statement in database:
if "in_response_to" in database[statement]:
in_response_to = database[statement]["in_response_to"]
# check if our closest statement is in this list
if closest_statement_key in in_response_to:
matching_responces.append(statement)
# If no matching responses are found: return something random
if not matching_responces:
selection = random.choice(list(database.keys()))
return {selection: database[selection]}
# Choose from the selection of matching responces
selection = random.choice(matching_responces)
return {selection: database[selection]}
def main():
# Create tmp db file in /tmp
#db = jsondb.create({})
# Create sqlite database
#db = jsondb.create({}, url='data.db')
# load database
db = Database("data.json")
db.data(key="user_count", value=241)
data = {
"user_id": 234565,
"user_name": "AwesomeUserName",
"is_moderator": True,
}
db.data(dictionary=data)
print db.data()
def test_create_none_exists(self):
"""
A file should be created if none exists
and an empty {} is placed in the file.
"""
from os import path
import os
database = Database("new.db")
data = ""
with open("new.db", "r") as db:
data = db.read()
self.assertTrue(path.exists("new.db"))
self.assertTrue("{}" in data)
os.remove("new.db")
def test_empty_file_initialized(self):
"""
Test that when a file exists but is empty,
a {} is added.
"""
import os
# Create an new empty file
open("empty.db", 'a').close()
database = Database("empty.db")
data = ""
with open("empty.db", "r") as db:
data = db.read()
self.assertTrue("{}" in data)
os.remove("empty.db")
app.add_url_rule("/legs/<string:leg_name>/hip/", view_func=Hip.as_view("hip"))
app.add_url_rule("/legs/<string:leg_name>/knee/", view_func=Knee.as_view("knee"))
app.add_url_rule("/legs/<string:leg_name>/ankle/", view_func=Ankle.as_view("ankle"))
app.add_url_rule("/neck/", view_func=Neck.as_view("neck"))
app.add_url_rule("/torso/", view_func=Torso.as_view("torso"))
app.add_url_rule("/api/terminate/",view_func=api.Terminate.as_view("terminate"))
app.add_url_rule("/api/speech/", view_func=api.Speech.as_view("speech"))
app.add_url_rule("/api/writing/", view_func=api.Writing.as_view("writing"))
app.add_url_rule("/api/chat/", view_func=api.Chat.as_view("chat"))
app.add_url_rule("/api/status/", view_func=api.Status.as_view("status"))
if __name__ == "__main__":
app.config["CHATBOT"] = ChatBot()
app.config["DATABASE"] = Database("settings.db")
app.config["DEBUG"] = True
app.config["SECRET_KEY"] = "development"
if settings_available:
if hasattr(settings, "GITHUB"):
app.config["GITHUB"] = GitHub(settings.GITHUB)
if hasattr(settings, "TWITTER"):
app.config["TWITTER"] = Twitter(settings.TWITTER)
if hasattr(settings, "GOOGLE"):
app.config["GOOGLE"] = settings.GOOGLE
if hasattr(settings, "DISQUS"):
app.config["DISQUS"] = settings.DISQUS
app.run(host="0.0.0.0", port=8000)
from wcrawler import *
from jsondb.db import Database
if __name__ == '__main__':
crawler = WCrawler(cookie = 'UOR=www.baidu.com,blog.sina.com.cn,; U_TRS1=0000004f.32c845f.5733e2d5.ad9108a6; U_TRS2=0000004f.32d445f.5733e2d5.b5716a19; SINAGLOBAL=61.135.169.79_1463018197.192534; Apache=61.135.169.79_1463018197.747655; SessionID=le113sggcdf7l11jsc6ohnlbu6; vjuids=4315adb9d.154a2ae19f0.0.b49484ff; ULV=1463038075786:2:2:2:61.135.169.79_1463018197.747655:1463018197886; SGUID=1464491184077_71168201; SCF=AqWXSdmqnlZXOZ97qJXfDPFH_LP7xOPjPIZ2KgmJLs5u4s9NUbHH-hKLtGyy6BSDgQjhOgDOgUaK_NW8PGgl_Js.; lxlrtst=1476689468_o; lxlrttp=1477565169; sso_info=v02m6alo5qztaSYlrmunpKZtZqWkL2Ms4C4jaOEtoyzlLWMgMDA; SINABLOGNUINFO=3086163550.b7f31e5e.; vjlast=1477628269.1478053154.11; tgc=TGT-MzA4NjE2MzU1MA==-1478097882-xd-C3E6ED5F3591D09ECDBF1C1870E7F2D7; SUB=_2A251HYuKDeTxGeVO41QQ9i3JzjyIHXVWavpCrDV_PUNbm9AKLUbFkW8Nc4Jp5aiyZ60bFPZlskjv-8lyRA..; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W5aIE.ERRyKAcZffYlVQ4195NHD95Q0ehnceKq0SK-7Ws4Dqcji-GS4-GLXIsvV; ALC=ac%3D2%26bt%3D1478097882%26cv%3D5.0%26et%3D1509633882%26scf%3DAqWXSdmqnlZXOZ97qJXfDPFH_LP7xOPjPIZ2KgmJLs5u4s9NUbHH-hKLtGyy6BSDgQjhOgDOgUaK_NW8PGgl_Js.%26uid%3D3086163550%26vf%3D0%26vs%3D0%26vt%3D0%26es%3Da2a19c21b755058448fa2428ee17b3fe; ALF=1509633882; LT=1478097882',\
max_num_weibo = 100, \
max_num_fans = 100, \
max_num_follow = 100, \
wfilter = 'all', \
return_type = 'string')
luyi_data = crawler.crawl(url='http://weibo.cn/luyi')
db = Database("luyi_data.db")
db.data(key="luyi_data", value=luyi_data)
''' TODO: Font support for Chinese '''
print(db.data(key="luyi_data"))
class JsonDatabaseAdapter(DatabaseAdapter):
def __init__(self, **kwargs):
super(JsonDatabaseAdapter, self).__init__(**kwargs)
database_path = self.kwargs.get("database", "database.db")
self.database = Database(database_path)
def _keys(self):
# The value has to be cast as a list for Python 3 compatibility
return list(self.database[0].keys())
def count(self):
return len(self._keys())
def find(self, statement_text):
values = self.database.data(key=statement_text)
if not values:
return None
return Statement(statement_text, **values)
def filter(self, **kwargs):
"""
Returns a list of statements in the database
that match the parameters specified.
"""
pass
# TODO: Useful for in_response_to...
def update(self, statement):
# Do not alter the database unless writing is enabled
if not self.read_only:
data = statement.serialize()
# Remove the text key from the data
del(data['text'])
self.database[statement.text] = data
# Make sure that an entry for each response is saved
for response_statement in statement.in_response_to:
response = self.find(response_statement)
if not response:
response = Statement(response_statement)
self.update(response)
return statement
def get_random(self):
from random import choice
if self.count() < 1:
raise EmptyDatabaseException()
statement = choice(self._keys())
return self.find(statement)
def drop(self):
"""
Remove the json file database completely.
"""
import os
os.remove(self.database.path)
def __init__(self, database_path):
self.database = Database(database_path)
from flask import Flask, render_template, request, jsonify
from raspberry import RaspberryThread
from time import sleep
from SmartSystem import SmartSystem
import datetime
import os
from jsondb.db import Database
from DatabaseFunctions import save
from Temperature import read_temp
import RPi.GPIO as GPIO
import sys
import signal
db = Database("data.db")
app = Flask(__name__)
# Создание обьекта и потока
System = SmartSystem()
Smart = RaspberryThread(function=System.loop)
# Коллекция потоков
threads = [
Smart,
]
# Берём данные с бд
System.times = db['system']['times']
System.Tens.TimeSleep = db['system']['timer']
System.temp = db['system']['temp']
# Вызов метода создания потока
def __init__(self):
self.db = Database(
join(
'database', 'Data_{}.db'.format(
datetime.datetime.now().strftime('%Y%m%d'))))
class JsonFileStorageAdapter(StorageAdapter):
"""
This adapter allows ChatterBot to store conversation
data in a file in JSON format.
"""
def __init__(self, **kwargs):
super(JsonFileStorageAdapter, self).__init__(**kwargs)
if not kwargs.get('silence_performance_warning', False):
warnings.warn(
'The JsonFileStorageAdapter is not recommended for production application environments.',
self.UnsuitableForProductionWarning)
database_path = self.kwargs.get('database', 'database.db')
self.database = Database(database_path)
self.adapter_supports_queries = False
def _keys(self):
# The value has to be cast as a list for Python 3 compatibility
return list(self.database[0].keys())
def count(self):
return len(self._keys())
def find(self, statement_text):
values = self.database.data(key=statement_text)
if not values:
return None
values['text'] = statement_text
return self.json_to_object(values)
def remove(self, statement_text):
"""
Removes the statement that matches the input text.
Removes any responses from statements if the response text matches the
input text.
"""
for statement in self.filter(in_response_to__contains=statement_text):
statement.remove_response(statement_text)
self.update(statement)
self.database.delete(statement_text)
def deserialize_responses(self, response_list):
"""
Takes the list of response items and returns
the list converted to Response objects.
"""
proxy_statement = Statement('')
for response in response_list:
data = response.copy()
text = data['text']
del (data['text'])
proxy_statement.add_response(Response(text, **data))
return proxy_statement.in_response_to
def json_to_object(self, statement_data):
# Don't modify the referenced object
statement_data = statement_data.copy()
# Build the objects for the response list
statement_data['in_response_to'] = self.deserialize_responses(
statement_data['in_response_to'])
# Remove the text attribute from the values
text = statement_data.pop('text')
return Statement(text, **statement_data)
def _all_kwargs_match_values(self, kwarguments, values):
for kwarg in kwarguments:
if '__' in kwarg:
kwarg_parts = kwarg.split('__')
key = kwarg_parts[0]
identifier = kwarg_parts[1]
if identifier == 'contains':
text_values = []
for val in values[key]:
text_values.append(val['text'])
if (kwarguments[kwarg]
not in text_values) and (kwarguments[kwarg]
not in values[key]):
return False
if kwarg in values:
if values[kwarg] != kwarguments[kwarg]:
return False
return True
def filter(self, **kwargs):
"""
Returns a list of statements in the database
that match the parameters specified.
"""
results = []
for key in self._keys():
values = self.database.data(key=key)
# Add the text attribute to the values
values['text'] = key
if self._all_kwargs_match_values(kwargs, values):
results.append(self.json_to_object(values))
return results
def update(self, statement, **kwargs):
# Do not alter the database unless writing is enabled
if not self.read_only:
data = statement.serialize()
# Remove the text key from the data
del (data['text'])
self.database.data(key=statement.text, value=data)
# Make sure that an entry for each response exists
for response_statement in statement.in_response_to:
response = self.find(response_statement.text)
if not response:
response = Statement(response_statement.text)
self.update(response)
return statement
def get_random(self):
from random import choice
if self.count() < 1:
raise self.EmptyDatabaseException()
statement = choice(self._keys())
return self.find(statement)
def drop(self):
"""
Remove the json file database completely.
"""
import os
if os.path.exists(self.database.path):
os.remove(self.database.path)
class UnsuitableForProductionWarning(Warning):
pass
class JsonDatabaseAdapter(StorageAdapter):
"""
The JsonDatabaseAdapter is an interface that allows chatbot
to store the conversation as a Json-encoded file.
"""
def __init__(self, **kwargs):
super(JsonDatabaseAdapter, self).__init__(**kwargs)
database_path = self.kwargs.get("database", "database.db")
self.database = Database(database_path)
def _keys(self):
# The value has to be cast as a list for Python 3 compatibility
return list(self.database[0].keys())
def count(self):
return len(self._keys())
def find(self, statement_text):
values = self.database.data(key=statement_text)
if not values:
return None
# Build the objects for the response list
response_list = self.deserialize_responses(values["in_response_to"])
values["in_response_to"] = response_list
return Statement(statement_text, **values)
def remove(self, statement_text):
"""
Removes the statement that matches the input text.
Removes any responses from statements if the response text matches the
input text.
"""
for statement in self.filter(in_response_to__contains=statement_text):
statement.remove_response(statement_text)
self.update(statement)
self.database.delete(statement_text)
def deserialize_responses(self, response_list):
"""
Takes the list of response items and returns the
list converted to object versions of the responses.
"""
in_response_to = []
for response in response_list:
text = response["text"]
del(response["text"])
in_response_to.append(
Response(text, **response)
)
return in_response_to
def _all_kwargs_match_values(self, kwarguments, values):
for kwarg in kwarguments:
if "__" in kwarg:
kwarg_parts = kwarg.split("__")
key = kwarg_parts[0]
identifier = kwarg_parts[1]
if identifier == "contains":
text_values = []
for val in values[key]:
text_values.append(val["text"])
if (kwarguments[kwarg] not in text_values) and (
kwarguments[kwarg] not in values[key]):
return False
if kwarg in values:
if values[kwarg] != kwarguments[kwarg]:
return False
return True
def filter(self, **kwargs):
"""
Returns a list of statements in the database
that match the parameters specified.
"""
results = []
for key in self._keys():
values = self.database.data(key=key)
# Add the text attribute to the values
values["text"] = key
if self._all_kwargs_match_values(kwargs, values):
# Build the objects for the response list
in_response_to = values["in_response_to"]
response_list = self.deserialize_responses(in_response_to)
values["in_response_to"] = response_list
# Remove the text attribute from the values
text = values.pop("text")
results.append(
Statement(text, **values)
)
return results
def update(self, statement):
# Do not alter the database unless writing is enabled
if not self.read_only:
data = statement.serialize()
# Remove the text key from the data
del(data['text'])
self.database.data(key=statement.text, value=data)
# Make sure that an entry for each response exists
for response_statement in statement.in_response_to:
response = self.find(response_statement.text)
if not response:
response = Statement(response_statement.text)
self.update(response)
return statement
def get_random(self):
from random import choice
if self.count() < 1:
raise EmptyDatabaseException()
statement = choice(self._keys())
return self.find(statement)
def drop(self):
"""
Remove the json file database completely.
"""
import os
if os.path.exists(self.database.path):
os.remove(self.database.path)
def loadDatabases(self):
self.db['groups'] = Database("cache/db/groups.json")
self.db['maps'] = TinyDB('cache/db/maps.json')
self.db['ranks'] = TinyDB('cache/db/ranks.json',
sort_keys=True,
indent=4)
class JsonDatabaseAdapter(DatabaseAdapter):
def __init__(self, **kwargs):
super(JsonDatabaseAdapter, self).__init__(**kwargs)
database_path = self.kwargs.get("database", "database.db")
self.database = Database(database_path)
def _keys(self):
# The value has to be cast as a list for Python 3 compatibility
return list(self.database[0].keys())
def count(self):
return len(self._keys())
def find(self, statement_text):
values = self.database.data(key=statement_text)
if not values:
return None
return Statement(statement_text, **values)
def _all_kwargs_match_values(self, kwarguments, values):
for kwarg in kwarguments:
if "__" in kwarg:
kwarg_parts = kwarg.split("__")
if kwarg_parts[1] == "contains":
if kwarguments[kwarg] not in values[kwarg_parts[0]]:
return False
if kwarg in values:
if values[kwarg] != kwarguments[kwarg]:
return False
return True
def filter(self, **kwargs):
"""
Returns a list of statements in the database
that match the parameters specified.
"""
results = []
for key in self._keys():
values = self.database.data(key=key)
if self._all_kwargs_match_values(kwargs, values):
results.append(
Statement(key, **values)
)
return results
def update(self, statement):
# Do not alter the database unless writing is enabled
if not self.read_only:
data = statement.serialize()
# Remove the text key from the data
del(data['text'])
self.database[statement.text] = data
# Make sure that an entry for each response is saved
for response_statement in statement.in_response_to:
response = self.find(response_statement)
if not response:
response = Statement(response_statement)
self.update(response)
return statement
def get_random(self):
from random import choice
if self.count() < 1:
raise EmptyDatabaseException()
statement = choice(self._keys())
return self.find(statement)
def drop(self):
"""
Remove the json file database completely.
"""
import os
os.remove(self.database.path)
import txaio
import simplejson as json
from autobahn.asyncio.websocket import WebSocketServerProtocol, \
WebSocketServerFactory
from jsondb.db import Database
# Tell txaio to use asyncio, not Twisted.
txaio.use_asyncio()
# This holds clients and their unique ID's.
CLIENT_LIST = {}
# Our database.
DB_HANDLE = Database("db.json")
# Hash for authentication. By default the password is SHA256 of "31Seks31", but change this
# as soon as you've finished testing.
PASS_HASH = "e1ed02693ddaad40e1f2a249a0915b0bc694c4d20760ca1d95c045ede86ff8ba"
def check_hash(password):
"Checks if the hash given by client is valid."
return hashlib.sha256(password.encode('utf-8')).hexdigest() == PASS_HASH
#
# Category handling functions. These handle the connection with jsondatabase.
#
def get_categories():
if DB_HANDLE["categories"] is not None:
return DB_HANDLE["categories"]
# coding: utf-8
import sys
import random
from jsondb.db import Database
from colorama import init, Fore, Back, Style
from bot import *
if __name__ != "__main__":
sys.exit(1)
init(autoreset=True)
# load accounts database
DB_ACCOUNTS = Database("cache/db/accounts.json")
bots = []
try:
# Init bots
for user in DB_ACCOUNTS.data()['users']:
bot = Bot(user['nickname'], user['password'], user['server'],
user['group'])
if bot.isLogged:
bots.append(bot)
# break
# Heartbeat
while 1:
for bot in bots:
bot.hello()
bot.setPause()
print("Pause..")
def _update_last_digest_timestamp(path):
db = Database(path)
db.data(
key='last-digest-timestamp',
value=time.strftime('%Y-%m-%dT%H:%M:%S%z', time.localtime())
)
def setUp(self):
self.database = Database(None)
c = Fore.GREEN
elif prefix == '-':
c = Fore.RED
elif prefix == '!':
c = Fore.YELLOW
c = Style.BRIGHT + c
e = Style.RESET_ALL + Fore.RESET
if line:
print c + "[" + now.strftime(
"%Y-%m-%d %H:%M") + "][" + prefix + "] " + text + e
else:
print "[" + now.strftime(
"%Y-%m-%d %H:%M") + "][" + c + prefix + e + "] " + text
dbAccounts = Database("../cache/db/accounts.json")
users = []
log('?', "Login..", True)
for user in dbAccounts.data()['users']:
if user['server'] != serverID:
continue
handle = ThirdWorldWar(user['nickname'], user['password'], user['server'])
if not handle.rLogin():
log('-', "Login for user \"" + user['nickname'] + "\"", False)
continue
log('+', "Login for user \"" + user['nickname'] + "\"", False)
users.append(handle)
log('?', "Scanning..", True)
i = 0
def setUp(self):
self.database = Database(None)
def __init__(self, **kwargs):
super(JsonDatabaseAdapter, self).__init__(**kwargs)
database_path = self.kwargs.get("database", "database.db")
self.database = Database(database_path)
from wcrawler import *
from jsondb.db import Database
if __name__ == '__main__':
crawler = WCrawler(cookie = 'UOR=www.baidu.com,blog.sina.com.cn,; U_TRS1=0000004f.32c845f.5733e2d5.ad9108a6; U_TRS2=0000004f.32d445f.5733e2d5.b5716a19; SINAGLOBAL=61.135.169.79_1463018197.192534; Apache=61.135.169.79_1463018197.747655; SessionID=le113sggcdf7l11jsc6ohnlbu6; vjuids=4315adb9d.154a2ae19f0.0.b49484ff; ULV=1463038075786:2:2:2:61.135.169.79_1463018197.747655:1463018197886; SGUID=1464491184077_71168201; SCF=AqWXSdmqnlZXOZ97qJXfDPFH_LP7xOPjPIZ2KgmJLs5u4s9NUbHH-hKLtGyy6BSDgQjhOgDOgUaK_NW8PGgl_Js.; lxlrtst=1476689468_o; lxlrttp=1477565169; sso_info=v02m6alo5qztaSYlrmunpKZtZqWkL2Ms4C4jaOEtoyzlLWMgMDA; SINABLOGNUINFO=3086163550.b7f31e5e.; vjlast=1477628269.1478053154.11; tgc=TGT-MzA4NjE2MzU1MA==-1478097882-xd-C3E6ED5F3591D09ECDBF1C1870E7F2D7; SUB=_2A251HYuKDeTxGeVO41QQ9i3JzjyIHXVWavpCrDV_PUNbm9AKLUbFkW8Nc4Jp5aiyZ60bFPZlskjv-8lyRA..; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W5aIE.ERRyKAcZffYlVQ4195NHD95Q0ehnceKq0SK-7Ws4Dqcji-GS4-GLXIsvV; ALC=ac%3D2%26bt%3D1478097882%26cv%3D5.0%26et%3D1509633882%26scf%3DAqWXSdmqnlZXOZ97qJXfDPFH_LP7xOPjPIZ2KgmJLs5u4s9NUbHH-hKLtGyy6BSDgQjhOgDOgUaK_NW8PGgl_Js.%26uid%3D3086163550%26vf%3D0%26vs%3D0%26vt%3D0%26es%3Da2a19c21b755058448fa2428ee17b3fe; ALF=1509633882; LT=1478097882',\
max_num_weibo = 100, \
max_num_fans = 100, \
max_num_follow = 100, \
wfilter = 'all', \
return_type = 'string')
luyi_data = crawler.crawl(url = 'http://weibo.cn/luyi')
db = Database("luyi_data.db")
db.data(key = "luyi_data", value = luyi_data)
''' TODO: Font support for Chinese '''
print(db.data(key = "luyi_data"))
