def dns_delete(dns_id): new_dns = Data.get('dns')[:] new_dns.pop(dns_id) Data.set('dns', new_dns) Data.save() DNS.ask_for_reload() return bottle.redirect('/dns')
def routes_restart(): # Rebuild chains Proxy.reset() Proxy.build_all(Data.get('proxies')) try: Proxy.prepare(Data.get('routes')) Proxy.exec() return bottle.redirect('/routes') except Exception as e: return bottle.redirect('/routes?{0}'.format(urlencode({'msg': str(e)})))
def test_get_csv_should_return_200_when_correct_authorization_and_have_data_for_user_id_and_status_is_true_when_have_data(self): token = generate_jwt(self.user_id) data_id = 1 keyword = "test-keyword" total_adword = 1 total_link = 1 total_search_result = "about 1,000" html_code = "test-html-code" new_data = Data( file_id = self.file_id, id = data_id, keyword = keyword, total_adword = total_adword, total_link = total_link, total_search_result = total_search_result, html_code = html_code ) self.db_session.add(new_data) self.db_session.commit() with app.test_client() as client: result = client.get( '/csv', headers={"Authorization": token} ) assert result.status_code == 200 expected_result = [ [self.file_id, self.filename, self.keywords, ANY, True] ] assert json.loads(result.data) == expected_result
def scrape_data_from_google(file_id, keyword): try: chrome_options = webdriver.ChromeOptions() chrome_options.add_argument("--no-sandbox") chrome_options.add_argument("--headless") chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument( "user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" ) driver = webdriver.Chrome(chrome_options=chrome_options) driver.get(f"https://www.google.com/search?q={keyword}") content = driver.page_source soup = BeautifulSoup(content, "html.parser") total_adword = count_adword(soup) total_link = count_link(soup) total_search_result = get_total_search_result(soup) finally: driver.close() new_data = Data(file_id=file_id, keyword=keyword, total_adword=total_adword, total_link=total_link, total_search_result=total_search_result, html_code=soup.prettify()) db_session.add(new_data) db_session.commit() time.sleep(int(os.environ["SCRAPING_DELAY"]))
def test_get_data_report_should_return_200_when_file_id_exist(self): data_id = 2 keyword = "test-keyword-2" self.new_data = Data(file_id=self.file_id, id=data_id, keyword=keyword, total_adword=self.total_adword, total_link=self.total_link, total_search_result=self.total_search_result, html_code=self.html_code) self.db_session.add(self.new_data) self.db_session.commit() with app.test_client() as client: result = client.get('/data-report/1') assert result.status_code == 200 expected_data = [[ self.keyword, self.total_adword, self.total_link, self.total_search_result, self.html_code, self.file_id ], [ keyword, self.total_adword, self.total_link, self.total_search_result, self.html_code, self.file_id ]] assert (json.loads(result.data), expected_data)
def routes_add(): if bottle.request.forms.get('name') is not None: dns_entry = {} dns_entry['name'] = bottle.request.forms.get('name') dns_entry['type'] = bottle.request.forms.get('dtype') dns_entry['value'] = bottle.request.forms.get('value') if dns_entry['value'] is None or dns_entry['value'] == '': return bottle.redirect('/dns/add?{0}'.format(urlencode({'msg': 'Value field is mandatory'}))) new_dns = Data.get('dns')[:] new_dns.append(dns_entry) Data.set('dns', new_dns) Data.save() DNS.ask_for_reload() return bottle.redirect('/dns') return {"title": "DockerAnon | DNS", "msg": bottle.request.query.msg}
def insert(): if request.method == 'POST': nome = request.json['nome'] endereco = request.json['endereco'] telefone = request.json['telefone'] data = request.json['data'] status = request.json['status'] try: user = Data(nome, endereco, telefone, data, status) user.save() return jsonify({'ok': True}) except: return jsonify({'mensagem': 'Ocorreu um erro interno'}), 500
def input_data(session, point, data, definition, unit='mg', unit_type='mass'): res = session.query(Data_Def).filter(Data_Def.name == definition).first() if not res: print 'Data definition <%s> non existent, please check' % definition return d = Data() d.data_type_id = res.id d.data_point_id = point.id #Todo def SI_Unit(unit_type)-> returns e.g. 'kg' ### conversion from input unit to storage unit [SI] if unit != 'kg' and unit_type == 'mass': d_conv = data[1] d_conv *= RPhelper.convert2(unit, 'kg', 'mass') d.unit = 'kg' if unit != 'm' and unit_type == 'length': if definition == 'height': d_conv = data[2] if definition == 'diameter': d_conv = data[3] d_conv *= RPhelper.convert2(unit, 'm', 'length') d.unit = 'm' d.valA = d_conv d.physical_property = definition session.add(d) session.commit()
def routes(): route_list = [] for r_name, r_data in Data.get('routes').items(): route_list.append({"name": r_name, "network": r_data.get("network"), "proxy": r_data.get("proxy")}) return {"title": "DockerAnon | Routes", "route": route_list, "msg": bottle.request.query.msg}
def proxy(): proxy_list = [] for p_name, p_data in Data.get('proxies').items(): proxy_list.append({"name": p_name, "type": p_data.get('type'), "host": "{0}:{1}".format(p_data.get('host', '127.0.0.1'), p_data.get('port', 9050)), "proxy": p_data.get("proxy", "-")}) return {"title": "DockerAnon | Proxy", "proxy": proxy_list, "msg": bottle.request.query.msg}
def getData(): try: all_data = Data.search_all() except: return jsonify({'mensagem': 'Ocorreu um erro interno'}), 500 list_dict = [] for content in all_data: list_dict.append(content.toDict()) return {'dados': list_dict}, 200
def dns(): dns_list = [] i = 0 for d in Data.get('dns'): entry = d.copy() entry['id'] = 0 dns_list.append(entry) i += 1 return {"title": "DockerAnon | DNS", "dns": dns_list, "msg": bottle.request.query.msg}
class DataService(): def __init__(self, data, check_version=True): self.__data = Data(data, check_version) def generate_query(self): values = self.__data.get_all_row_values() schema = self.__data.get_schema() name = self.__data.get_name() output_formating = self.__data.get_output_formating() output_formating_keys = list(output_formating.keys()) query_list = [] for i in range(len(values)): _ql = [] for j in range(len(schema)): value = str(values[i][j]) substitute_value = None if schema[j] in output_formating_keys: substitute_value = output_formating[schema[j]] value = substitute_value.replace("#[value]", value) _ql.append(value + " AS " + str(schema[j])) query = ", ".join(_ql) query = "SELECT " + query query_list.append(query) query = " UNION ALL ".join(query_list) query = name + " AS ( " + query + " )" return query def get_name(self): return self.__data.get_name() def get_schema(self): return self.__data.get_schema()
def user_create(): if request.method == 'POST': name = request.form['user-name'] phone = request.form['phone-number'] try: data = Data(name=name, phone=phone) db.session.add(data) db.session.commit() return redirect('/users-list') except: return '<h2>DB create error</h2>' else: return render_template('create_user.html')
def proxy_add(): if bottle.request.forms.get('name') is not None: proxy = {} proxy['type'] = bottle.request.forms.get('ptype') if proxy['type'] is None: return bottle.redirect('/proxy/add?{0}'.format(urlencode({'msg': 'Type field is mandatory'}))) proxy['host'] = bottle.request.forms.get('host') if proxy['host'] is None: return bottle.redirect('/proxy/add?{0}'.format(urlencode({'msg': 'Host field is mandatory'}))) proxy['port'] = bottle.request.forms.get('port') if proxy['host'] is None: return bottle.redirect('/proxy/add?{0}'.format(urlencode({'msg': 'Port field is mandatory'}))) user = bottle.request.forms.get('user') if user is None and proxy['type'] == 'ssh': return bottle.redirect('/proxy/add?{0}'.format(urlencode({'msg': 'User field is mandatory for SSH'}))) elif user is not None: proxy['user'] = user password = bottle.request.forms.get('password') if password is not None: proxy['password'] = password sshkey = bottle.request.forms.get('sshkey') if sshkey is not None: proxy['sshkey'] = sshkey if proxy['type'] == 'ssh' and password is None and sshkey is None: return bottle.redirect('/proxy/add?{0}'.format(urlencode({'msg': 'You must provide password or SSH key for SSH'}))) previous = bottle.request.forms.get('previous') if previous != 'None': proxy['proxy'] = previous hostkeys = bottle.request.forms.get('hostkeys') if len(hostkeys) > 0: proxy['hostkeys'] = hostkeys.split('\n') new_proxies = Data.get('proxies').copy() new_proxies[bottle.request.forms.get('name')] = proxy # Rebuild chains Proxy.reset() Proxy.build_all(new_proxies) try: Proxy.prepare(Data.get('routes')) Data.set('proxies', new_proxies) Data.save() Proxy.exec() return bottle.redirect('/proxy') except Exception as e: return bottle.redirect('/proxy?{0}'.format(urlencode({'msg': str(e)}))) return {"title": "DockerAnon | Proxy", "proxy": list(Data.get('proxies').keys()), "msg": bottle.request.query.msg}
async def get(self): """ Handler for get requests, async load from database. """ result = [] name = self.request.match_info.get('name', None) if not name: logger.error('Missing request attribute "name"') raise web.HTTPBadRequest() data = await manager.execute(Data.select().where(Data.name == name)) for item in data: result.append(model_to_dict(item)) logger.info(f'Result contain: {len(data)} Data items by name: {name}') # lambda used to serialization datetime return web.json_response(data=result, dumps=lambda x: json.dumps(x, default=json_serial))
def proxy_delete(proxy_name): new_proxies = Data.get('proxies').copy() new_proxies.pop(proxy_name) # Rebuild chains Proxy.reset() Proxy.build_all(new_proxies) try: Proxy.prepare(Data.get('routes')) Data.set('proxies', new_proxies) Data.save() Proxy.exec() return bottle.redirect('/proxy') except Exception as e: return bottle.redirect('/proxy?{0}'.format(urlencode({'msg': str(e)})))
def init_data(): data = Data().data countries = [] types = [ {'label': "Confirmed", 'value': "confirmed"}, {'label': "Deaths", 'value': "deaths"}, ] tots = { 'last_date': None, 'confirmed': 0, 'deaths': 0, } for country in data: if len(data[country]) > 0: countries.append({'label': country, 'value': country}) tots['confirmed'] += data[country][-1].get('confirmed', 0) tots['deaths'] += data[country][-1].get('deaths', 0) tots['last_date'] = datetime.strptime(data[country][-1].get('date'), '%Y-%m-%d') return data, countries, types, tots
def setUp(self): super().setUp() self.user_id = 1 self.email = "[email protected]" self.password = "******" self.file_id = 1 self.filename = "test-file.csv" self.keywords = 1 self.data_id = 1 self.keyword = "test-keyword" self.total_adword = 1 self.total_link = 1 self.total_search_result = "about 1,000" self.html_code = "test-html-code" self.new_user = User(id=self.user_id, email=self.email, password=self.password) self.db_session.add(self.new_user) self.db_session.commit() self.new_file = File(user_id=self.user_id, id=self.file_id, filename=self.filename, keywords=self.keywords) self.db_session.add(self.new_file) self.db_session.commit() self.new_data = Data(file_id=self.file_id, id=self.data_id, keyword=self.keyword, total_adword=self.total_adword, total_link=self.total_link, total_search_result=self.total_search_result, html_code=self.html_code) self.db_session.add(self.new_data) self.db_session.commit()
def routes_add(): if bottle.request.forms.get('name') is not None: route = {} route['network'] = bottle.request.forms.get('network') if route['network'] is None: return bottle.redirect('/routes/add?{0}'.format(urlencode({'msg': 'Network field is mandatory'}))) route['proxy'] = bottle.request.forms.get('proxy') new_routes = Data.get('routes').copy() new_routes[bottle.request.forms.get('name')] = route # Rebuild chains Proxy.reset() Proxy.build_all(Data.get('proxies')) try: Proxy.prepare(new_routes) Data.set('routes', new_routes) Data.save() Proxy.exec() return bottle.redirect('/routes') except Exception as e: return bottle.redirect('/routes?{0}'.format(urlencode({'msg': str(e)}))) return {"title": "DockerAnon | Routes", "proxy": list(Data.get('proxies').keys()), "msg": bottle.request.query.msg}
import os from urllib.parse import urlencode import bottle from models.data import Data from models.proxy import Proxy from models.dns import DNSServer, UDPRequestHandler Data.load() DNS_HOST = os.getenv('GATEWAY_IP') DNS_FORWARDER = '127.0.0.1' DNS_FORWARDER_PORT = 5353 DNS = DNSServer((DNS_HOST, 53), UDPRequestHandler) # Launch DNS DNS.load_zone() DNS.set_forwarder(DNS_FORWARDER, 5353) DNS.start() # Launch Proxy Proxy.reset() Proxy.build_all(Data.get('proxies')) Proxy.prepare(Data.get('routes')) Proxy.exec() @bottle.route('/static/<filename:path>') def server_static(filename): return bottle.static_file(filename, root='static')
def __init__(self, data, check_version=True): self.__data = Data(data, check_version)
def Index(): all_data = Data.search_all() return render_template('index.html', dados=all_data)
async def test_bot(): config = Data(getenv('CONFIG_LOCT', default='config.yml')) data = Data(getenv('DATA_LOCT', default='data.yml')) bot = WatsonBot(config=config, data=data) await bot.login(config['bot']['token']) await bot.close()
def run(self): for _ in range(100): d = Data() d.field = 'Data %s' % random.randint(0, 100) db.session.add(d) db.session.commit()
#!/usr/bin/env python3 from models.data import Data from models.NeuralNetwork import NeuralNetwork as Model from models.eval import smape import matplotlib.pyplot as plt import numpy as np offset = 0 n_train = 10000 n_test = 500 data = Data('data/database.sqlite') data.read('May2015', ['body'], offset=offset) model = Model(data, n_train) predicted, actual = model.test(data, n_test) print('Smape: {}'.format(smape(predicted, actual))) print(n_train) print(n_test) if False: # These are some interesting visualizations possible with models that use the vectorizer l = np.array(["", "the", "reddit", "a", "dog"]) testX = model.vectorizer.transform(l) p = model.model.predict(testX) print(model.model.intercept_) print(p) print(l) plt.plot(predicted, actual, 'o') plt.ylim()
def main(): config = Data(getenv('CONFIG_LOCT', default='config.yml')) data = Data(getenv('DATA_LOCT', default='data.yml')) bot = WatsonBot(config=config, data=data) bot.run(config['bot']['token'])
# get envvars DEBUG = os.getenv('DEBUG', False) AUTH_USER = os.getenv('BASIC_AUTH_USERNAME', 'bober') AUTH_PASS = os.getenv('BASIC_AUTH_PASSWORD', 'pleasechange') # bootstrap the app app = Flask(__name__) # configure app before libraries app.config['BASIC_AUTH_USERNAME'] = AUTH_USER app.config['BASIC_AUTH_PASSWORD'] = AUTH_PASS # attach libraries Bootstrap(app) basic_auth = BasicAuth(app) store = Data() @app.route('/') def index(): deps = store.get_data() return render_template("index.jinja", releases=deps['releases'], kits=deps['kits']) @app.route('/update_git') @basic_auth.required def update_git(): store.update_versions() return "OK" @app.route('/update_version', methods=['POST']) @basic_auth.required def update_version():