def dns_delete(dns_id):
new_dns = Data.get('dns')[:]
new_dns.pop(dns_id)
Data.set('dns', new_dns)
Data.save()
DNS.ask_for_reload()
return bottle.redirect('/dns')
def routes_restart():
# Rebuild chains
Proxy.reset()
Proxy.build_all(Data.get('proxies'))
try:
Proxy.prepare(Data.get('routes'))
Proxy.exec()
return bottle.redirect('/routes')
except Exception as e:
return bottle.redirect('/routes?{0}'.format(urlencode({'msg': str(e)})))
def test_get_csv_should_return_200_when_correct_authorization_and_have_data_for_user_id_and_status_is_true_when_have_data(self):
token = generate_jwt(self.user_id)
data_id = 1
keyword = "test-keyword"
total_adword = 1
total_link = 1
total_search_result = "about 1,000"
html_code = "test-html-code"
new_data = Data(
file_id = self.file_id,
id = data_id,
keyword = keyword,
total_adword = total_adword,
total_link = total_link,
total_search_result = total_search_result,
html_code = html_code
)
self.db_session.add(new_data)
self.db_session.commit()
with app.test_client() as client:
result = client.get(
'/csv',
headers={"Authorization": token}
)
assert result.status_code == 200
expected_result = [
[self.file_id, self.filename, self.keywords, ANY, True]
]
assert json.loads(result.data) == expected_result
def scrape_data_from_google(file_id, keyword):
try:
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument(
"user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"
)
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.get(f"https://www.google.com/search?q={keyword}")
content = driver.page_source
soup = BeautifulSoup(content, "html.parser")
total_adword = count_adword(soup)
total_link = count_link(soup)
total_search_result = get_total_search_result(soup)
finally:
driver.close()
new_data = Data(file_id=file_id,
keyword=keyword,
total_adword=total_adword,
total_link=total_link,
total_search_result=total_search_result,
html_code=soup.prettify())
db_session.add(new_data)
db_session.commit()
time.sleep(int(os.environ["SCRAPING_DELAY"]))
def test_get_data_report_should_return_200_when_file_id_exist(self):
data_id = 2
keyword = "test-keyword-2"
self.new_data = Data(file_id=self.file_id,
id=data_id,
keyword=keyword,
total_adword=self.total_adword,
total_link=self.total_link,
total_search_result=self.total_search_result,
html_code=self.html_code)
self.db_session.add(self.new_data)
self.db_session.commit()
with app.test_client() as client:
result = client.get('/data-report/1')
assert result.status_code == 200
expected_data = [[
self.keyword, self.total_adword, self.total_link,
self.total_search_result, self.html_code, self.file_id
],
[
keyword, self.total_adword, self.total_link,
self.total_search_result, self.html_code,
self.file_id
]]
assert (json.loads(result.data), expected_data)
def routes_add():
if bottle.request.forms.get('name') is not None:
dns_entry = {}
dns_entry['name'] = bottle.request.forms.get('name')
dns_entry['type'] = bottle.request.forms.get('dtype')
dns_entry['value'] = bottle.request.forms.get('value')
if dns_entry['value'] is None or dns_entry['value'] == '':
return bottle.redirect('/dns/add?{0}'.format(urlencode({'msg': 'Value field is mandatory'})))
new_dns = Data.get('dns')[:]
new_dns.append(dns_entry)
Data.set('dns', new_dns)
Data.save()
DNS.ask_for_reload()
return bottle.redirect('/dns')
return {"title": "DockerAnon | DNS",
"msg": bottle.request.query.msg}
def insert():
if request.method == 'POST':
nome = request.json['nome']
endereco = request.json['endereco']
telefone = request.json['telefone']
data = request.json['data']
status = request.json['status']
try:
user = Data(nome, endereco, telefone, data, status)
user.save()
return jsonify({'ok': True})
except:
return jsonify({'mensagem': 'Ocorreu um erro interno'}), 500
def input_data(session, point, data, definition, unit='mg', unit_type='mass'):
res = session.query(Data_Def).filter(Data_Def.name == definition).first()
if not res:
print 'Data definition <%s> non existent, please check' % definition
return
d = Data()
d.data_type_id = res.id
d.data_point_id = point.id
#Todo def SI_Unit(unit_type)-> returns e.g. 'kg'
### conversion from input unit to storage unit [SI]
if unit != 'kg' and unit_type == 'mass':
d_conv = data[1]
d_conv *= RPhelper.convert2(unit, 'kg', 'mass')
d.unit = 'kg'
if unit != 'm' and unit_type == 'length':
if definition == 'height':
d_conv = data[2]
if definition == 'diameter':
d_conv = data[3]
d_conv *= RPhelper.convert2(unit, 'm', 'length')
d.unit = 'm'
d.valA = d_conv
d.physical_property = definition
session.add(d)
session.commit()
def routes():
route_list = []
for r_name, r_data in Data.get('routes').items():
route_list.append({"name": r_name,
"network": r_data.get("network"),
"proxy": r_data.get("proxy")})
return {"title": "DockerAnon | Routes",
"route": route_list,
"msg": bottle.request.query.msg}
def proxy():
proxy_list = []
for p_name, p_data in Data.get('proxies').items():
proxy_list.append({"name": p_name,
"type": p_data.get('type'),
"host": "{0}:{1}".format(p_data.get('host', '127.0.0.1'), p_data.get('port', 9050)),
"proxy": p_data.get("proxy", "-")})
return {"title": "DockerAnon | Proxy",
"proxy": proxy_list,
"msg": bottle.request.query.msg}
def getData():
try:
all_data = Data.search_all()
except:
return jsonify({'mensagem': 'Ocorreu um erro interno'}), 500
list_dict = []
for content in all_data:
list_dict.append(content.toDict())
return {'dados': list_dict}, 200
def dns():
dns_list = []
i = 0
for d in Data.get('dns'):
entry = d.copy()
entry['id'] = 0
dns_list.append(entry)
i += 1
return {"title": "DockerAnon | DNS",
"dns": dns_list,
"msg": bottle.request.query.msg}
class DataService():
def __init__(self, data, check_version=True):
self.__data = Data(data, check_version)
def generate_query(self):
values = self.__data.get_all_row_values()
schema = self.__data.get_schema()
name = self.__data.get_name()
output_formating = self.__data.get_output_formating()
output_formating_keys = list(output_formating.keys())
query_list = []
for i in range(len(values)):
_ql = []
for j in range(len(schema)):
value = str(values[i][j])
substitute_value = None
if schema[j] in output_formating_keys:
substitute_value = output_formating[schema[j]]
value = substitute_value.replace("#[value]", value)
_ql.append(value + " AS " + str(schema[j]))
query = ", ".join(_ql)
query = "SELECT " + query
query_list.append(query)
query = " UNION ALL ".join(query_list)
query = name + " AS ( " + query + " )"
return query
def get_name(self):
return self.__data.get_name()
def get_schema(self):
return self.__data.get_schema()
def user_create():
if request.method == 'POST':
name = request.form['user-name']
phone = request.form['phone-number']
try:
data = Data(name=name, phone=phone)
db.session.add(data)
db.session.commit()
return redirect('/users-list')
except:
return '<h2>DB create error</h2>'
else:
return render_template('create_user.html')
def proxy_add():
if bottle.request.forms.get('name') is not None:
proxy = {}
proxy['type'] = bottle.request.forms.get('ptype')
if proxy['type'] is None:
return bottle.redirect('/proxy/add?{0}'.format(urlencode({'msg': 'Type field is mandatory'})))
proxy['host'] = bottle.request.forms.get('host')
if proxy['host'] is None:
return bottle.redirect('/proxy/add?{0}'.format(urlencode({'msg': 'Host field is mandatory'})))
proxy['port'] = bottle.request.forms.get('port')
if proxy['host'] is None:
return bottle.redirect('/proxy/add?{0}'.format(urlencode({'msg': 'Port field is mandatory'})))
user = bottle.request.forms.get('user')
if user is None and proxy['type'] == 'ssh':
return bottle.redirect('/proxy/add?{0}'.format(urlencode({'msg': 'User field is mandatory for SSH'})))
elif user is not None:
proxy['user'] = user
password = bottle.request.forms.get('password')
if password is not None:
proxy['password'] = password
sshkey = bottle.request.forms.get('sshkey')
if sshkey is not None:
proxy['sshkey'] = sshkey
if proxy['type'] == 'ssh' and password is None and sshkey is None:
return bottle.redirect('/proxy/add?{0}'.format(urlencode({'msg': 'You must provide password or SSH key for SSH'})))
previous = bottle.request.forms.get('previous')
if previous != 'None':
proxy['proxy'] = previous
hostkeys = bottle.request.forms.get('hostkeys')
if len(hostkeys) > 0:
proxy['hostkeys'] = hostkeys.split('\n')
new_proxies = Data.get('proxies').copy()
new_proxies[bottle.request.forms.get('name')] = proxy
# Rebuild chains
Proxy.reset()
Proxy.build_all(new_proxies)
try:
Proxy.prepare(Data.get('routes'))
Data.set('proxies', new_proxies)
Data.save()
Proxy.exec()
return bottle.redirect('/proxy')
except Exception as e:
return bottle.redirect('/proxy?{0}'.format(urlencode({'msg': str(e)})))
return {"title": "DockerAnon | Proxy",
"proxy": list(Data.get('proxies').keys()),
"msg": bottle.request.query.msg}
async def get(self):
"""
Handler for get requests, async load from database.
"""
result = []
name = self.request.match_info.get('name', None)
if not name:
logger.error('Missing request attribute "name"')
raise web.HTTPBadRequest()
data = await manager.execute(Data.select().where(Data.name == name))
for item in data:
result.append(model_to_dict(item))
logger.info(f'Result contain: {len(data)} Data items by name: {name}')
# lambda used to serialization datetime
return web.json_response(data=result, dumps=lambda x: json.dumps(x, default=json_serial))
def proxy_delete(proxy_name):
new_proxies = Data.get('proxies').copy()
new_proxies.pop(proxy_name)
# Rebuild chains
Proxy.reset()
Proxy.build_all(new_proxies)
try:
Proxy.prepare(Data.get('routes'))
Data.set('proxies', new_proxies)
Data.save()
Proxy.exec()
return bottle.redirect('/proxy')
except Exception as e:
return bottle.redirect('/proxy?{0}'.format(urlencode({'msg': str(e)})))
def init_data():
data = Data().data
countries = []
types = [
{'label': "Confirmed", 'value': "confirmed"},
{'label': "Deaths", 'value': "deaths"},
]
tots = {
'last_date': None,
'confirmed': 0,
'deaths': 0,
}
for country in data:
if len(data[country]) > 0:
countries.append({'label': country, 'value': country})
tots['confirmed'] += data[country][-1].get('confirmed', 0)
tots['deaths'] += data[country][-1].get('deaths', 0)
tots['last_date'] = datetime.strptime(data[country][-1].get('date'), '%Y-%m-%d')
return data, countries, types, tots
def setUp(self):
super().setUp()
self.user_id = 1
self.email = "[email protected]"
self.password = "******"
self.file_id = 1
self.filename = "test-file.csv"
self.keywords = 1
self.data_id = 1
self.keyword = "test-keyword"
self.total_adword = 1
self.total_link = 1
self.total_search_result = "about 1,000"
self.html_code = "test-html-code"
self.new_user = User(id=self.user_id,
email=self.email,
password=self.password)
self.db_session.add(self.new_user)
self.db_session.commit()
self.new_file = File(user_id=self.user_id,
id=self.file_id,
filename=self.filename,
keywords=self.keywords)
self.db_session.add(self.new_file)
self.db_session.commit()
self.new_data = Data(file_id=self.file_id,
id=self.data_id,
keyword=self.keyword,
total_adword=self.total_adword,
total_link=self.total_link,
total_search_result=self.total_search_result,
html_code=self.html_code)
self.db_session.add(self.new_data)
self.db_session.commit()
def routes_add():
if bottle.request.forms.get('name') is not None:
route = {}
route['network'] = bottle.request.forms.get('network')
if route['network'] is None:
return bottle.redirect('/routes/add?{0}'.format(urlencode({'msg': 'Network field is mandatory'})))
route['proxy'] = bottle.request.forms.get('proxy')
new_routes = Data.get('routes').copy()
new_routes[bottle.request.forms.get('name')] = route
# Rebuild chains
Proxy.reset()
Proxy.build_all(Data.get('proxies'))
try:
Proxy.prepare(new_routes)
Data.set('routes', new_routes)
Data.save()
Proxy.exec()
return bottle.redirect('/routes')
except Exception as e:
return bottle.redirect('/routes?{0}'.format(urlencode({'msg': str(e)})))
return {"title": "DockerAnon | Routes",
"proxy": list(Data.get('proxies').keys()),
"msg": bottle.request.query.msg}
import os
from urllib.parse import urlencode
import bottle
from models.data import Data
from models.proxy import Proxy
from models.dns import DNSServer, UDPRequestHandler
Data.load()
DNS_HOST = os.getenv('GATEWAY_IP')
DNS_FORWARDER = '127.0.0.1'
DNS_FORWARDER_PORT = 5353
DNS = DNSServer((DNS_HOST, 53), UDPRequestHandler)
# Launch DNS
DNS.load_zone()
DNS.set_forwarder(DNS_FORWARDER, 5353)
DNS.start()
# Launch Proxy
Proxy.reset()
Proxy.build_all(Data.get('proxies'))
Proxy.prepare(Data.get('routes'))
Proxy.exec()
@bottle.route('/static/<filename:path>')
def server_static(filename):
return bottle.static_file(filename, root='static')
def __init__(self, data, check_version=True):
self.__data = Data(data, check_version)
def Index():
all_data = Data.search_all()
return render_template('index.html', dados=all_data)
async def test_bot():
config = Data(getenv('CONFIG_LOCT', default='config.yml'))
data = Data(getenv('DATA_LOCT', default='data.yml'))
bot = WatsonBot(config=config, data=data)
await bot.login(config['bot']['token'])
await bot.close()
def run(self):
for _ in range(100):
d = Data()
d.field = 'Data %s' % random.randint(0, 100)
db.session.add(d)
db.session.commit()
#!/usr/bin/env python3
from models.data import Data
from models.NeuralNetwork import NeuralNetwork as Model
from models.eval import smape
import matplotlib.pyplot as plt
import numpy as np
offset = 0
n_train = 10000
n_test = 500
data = Data('data/database.sqlite')
data.read('May2015', ['body'], offset=offset)
model = Model(data, n_train)
predicted, actual = model.test(data, n_test)
print('Smape: {}'.format(smape(predicted, actual)))
print(n_train)
print(n_test)
if False:
# These are some interesting visualizations possible with models that use the vectorizer
l = np.array(["", "the", "reddit", "a", "dog"])
testX = model.vectorizer.transform(l)
p = model.model.predict(testX)
print(model.model.intercept_)
print(p)
print(l)
plt.plot(predicted, actual, 'o')
plt.ylim()
def main():
config = Data(getenv('CONFIG_LOCT', default='config.yml'))
data = Data(getenv('DATA_LOCT', default='data.yml'))
bot = WatsonBot(config=config, data=data)
bot.run(config['bot']['token'])
# get envvars
DEBUG = os.getenv('DEBUG', False)
AUTH_USER = os.getenv('BASIC_AUTH_USERNAME', 'bober')
AUTH_PASS = os.getenv('BASIC_AUTH_PASSWORD', 'pleasechange')
# bootstrap the app
app = Flask(__name__)
# configure app before libraries
app.config['BASIC_AUTH_USERNAME'] = AUTH_USER
app.config['BASIC_AUTH_PASSWORD'] = AUTH_PASS
# attach libraries
Bootstrap(app)
basic_auth = BasicAuth(app)
store = Data()
@app.route('/')
def index():
deps = store.get_data()
return render_template("index.jinja", releases=deps['releases'], kits=deps['kits'])
@app.route('/update_git')
@basic_auth.required
def update_git():
store.update_versions()
return "OK"
@app.route('/update_version', methods=['POST'])
@basic_auth.required
def update_version():