def __init__(self, addr, port): RPCServer.__init__(self, addr, port) len_up = len(UPLOAD_SERVERS) len_repo = len(REPOSITORY_SERVERS) if len_up < len_repo or len_up % len_repo != 0: show_error(self, 'failed to initialize') raise Exception('failed to initialize') addr = localhost() if addr not in REPOSITORY_SERVERS: show_error(self, 'failed to initialize') raise Exception('failed to initialize REPOSITORY_SERVERS') for i in range(len(REPOSITORY_SERVERS)): if addr == REPOSITORY_SERVERS[i]: break total = len_up / len_repo self._upload_servers = UPLOAD_SERVERS[i * total:(i + 1) * total] self._print('upload_servers=%s' % str(self._upload_servers)) if HDFS: self._port = HDFS_PORT self._client = HDFSClient() else: self._port = FTP_PORT self._client = FTPClient() self._server = FTPServer() if REPO_DB: self._db = Database(addr=REPO_DB) else: self._db = Database(addr=addr) locks = [] for _ in range(LOCK_MAX): locks.append(Lock()) self._locks = HashRing(locks) if DEBUG: self._upload_cnt = 0 self._download_cnt = 0
def prepare(self, IP): self.db = Database(DBPATH) self.ip = IP self.zombie = sina_zombie(IP) zombie_info = self.db.get_zombie(0, 1, now())[0] self.upass = zombie_info[2].strip('\r').strip('\n') self.uname = zombie_info[1] self.uid = zombie_info[0]
def post(self): parser = reqparse.RequestParser() parser.add_argument('code', required=True, help='code can not be blank!') args = parser.parse_args() code = args['code'] # LINE notify json data client = { 'grant_type': 'authorization_code', 'code': code, 'redirect_uri': os.getenv("NOTIFY_REDIRECT_URI"), 'client_id': os.getenv("NOTIFY_CLIENT_ID"), 'client_secret': os.getenv("NOTIFY_CLIENT_SECRET") } # send request to auth r = requests.post('https://notify-bot.line.me/oauth/token', data=client) req = json.loads(r.text) if req['status'] == 200: token = req['access_token'] # Here is use PostgreSQL, you can change your love db with Database() as db, db.connect() as conn: with conn.cursor( cursor_factory=psycopg2.extras.RealDictCursor) as cur: cur.execute( f"INSERT INTO notify(token) VALUES ('{token}')") return {'access_token': req['access_token']}, 200 else: return {'message': r.text}, 200
def test_init(self, db): assert db is not None with pytest.raises(Exception) as exec_info: db = Database("/usr/bin/db.json") assert "not permitted" in str( exec_info.value) or "Permission denied" in str(exec_info.value)
def prepare(self,IP): self.db = Database(DBPATH) self.ip = IP self.zombie = sina_zombie(IP) zombie_info = self.db.get_zombie(0,1,now())[0] self.upass = zombie_info[2].strip('\r').strip('\n') self.uname = zombie_info[1] self.uid = zombie_info[0]
def get(self): msg = request.args.get('msg') with Database() as db, db.connect() as conn, conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: cur.execute("SELECT * FROM users") fetch = cur.fetchall() for user in fetch: # LINE push message self.line_bot_api.push_message( user['id'], TextSendMessage(text=msg)) return {'message': msg}, 200
def main(): db = Database.get() db.create_tables([SynthButtonSetting, WavFile]) drivers = load_drivers(config.SAMPLE_RATE) with AudioManager(drivers) as audio_manager: app = generate_flask_app(audio_manager) http_server = WSGIServer(("", 3001), app) signal.signal(signal.SIGINT, lambda *args, **kwargs: http_server.stop()) http_server.serve_forever()
def run_parsing(config): parser = argparse.ArgumentParser() subparsers = parser.add_subparsers() # Search parser group search_parser = subparsers.add_parser('search') search_parser.add_argument('QUERY', action='store', type=str, default=False) search_parser.add_argument('--output', '-o', action='store', required=False, default='json', choices=['tsv', 'json', 'binary']) search_parser.add_argument('--columns', '-c', action='store', required=False, help='Coma separated names of columns') # Prune parser group prune_parser = subparsers.add_parser('prune') prune_parser.add_argument('PRUNE', action='store', type=int, default=False) args = parser.parse_args() try: db_path = os.environ['DB_PATH'] except KeyError: print("Env variable DB_PATH is missing!", file=sys.stderr) exit(1) db = Database(db_path) if 'QUERY' in args: try: # Scrape and download down = Downloader() parser = Parser(config) d = Driver(config, args.QUERY, down, parser, db) torrents, headers = d.run() # Format and print fm = lib.utils.Formatter(headers, torrents) if args.columns: fm.filter_data(args.columns.split(',')) fm.format_data(args.output) fm.print_data() except: traceback.print_exc() exit(1) elif 'PRUNE' in args: try: # Prune DB db.connect() db.prune(args.PRUNE, time.time()) except: traceback.print_exc() exit(1)
class Config(object): def __init__(self, **kwargs): super(self.__class__, self).__init__(**kwargs) self.db = Database() def set(self, key, value): results = self.db.find_one('config', {'key': key}) if results is not None: self.db.delete(results) self.db.insert('config', {'key': key, 'value': value}) return 'Config set: ' + key + ' = ' + value def get(self, key): results = self.db.find_one('config', {'key': key}) if results is None: raise ValueError(key + ' is not set in config') return None return results['value'] def request(self, key): # pragma: no cover if sys.version_info < (3, 0): value = raw_input("Enter value for config '" + key + "': ") else: value = input("Enter value for config '" + key + "': ") self.set(key, value) return value def get_or_request(self, key): try: value = self.get(key) except ValueError: value = self.request(key) return value
class Thread_Zombie(threading.Thread): """docstring for ClassName""" def prepare(self,IP): self.db = Database(DBPATH) self.ip = IP self.zombie = sina_zombie(IP) zombie_info = self.db.get_zombie(0,1,now())[0] self.upass = zombie_info[2].strip('\r').strip('\n') self.uname = zombie_info[1] self.uid = zombie_info[0] def run(self): zombie_routine(self.zombie,self.uname,self.upass,self.uid,self.db) print self.ip,self.uname,self.upass,self.uid
def __init__(self, **kwargs): self.act = {} self.db = kwargs.get('db', Database()) self.dir = kwargs.get('dir', None) self.files = self.dir + '/files' self.config = kwargs.get('config', None) self.collection = '' self.match = None self.execution_order = 1 self.logging = kwargs.get('logging', None) self.history = {} self.idle_methods = [] self.define_idle(self.export_db, 24, lib.dt.datetime_from_time(0, 0)) # export db nightly
class Thread_Zombie(threading.Thread): """docstring for ClassName""" def prepare(self, IP): self.db = Database(DBPATH) self.ip = IP self.zombie = sina_zombie(IP) zombie_info = self.db.get_zombie(0, 1, now())[0] self.upass = zombie_info[2].strip('\r').strip('\n') self.uname = zombie_info[1] self.uid = zombie_info[0] def run(self): zombie_routine(self.zombie, self.uname, self.upass, self.uid, self.db) print self.ip, self.uname, self.upass, self.uid
def get(self): msg = request.args.get('msg') with Database() as db, db.connect() as conn: with conn.cursor( cursor_factory=psycopg2.extras.RealDictCursor) as cur: cur.execute(f"SELECT token FROM notify") fetch = cur.fetchall() for f in fetch: headers = { 'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': f"Bearer {f['token']}" } payload = {'message': msg} r = requests.post('https://notify-api.line.me/api/notify', data=payload, headers=headers) return {'result': 'ok'}, 200
def post(self): parser = reqparse.RequestParser() parser.add_argument('message', required=True, help='message can not be blank!') args = parser.parse_args() msg = args['message'] with Database() as db, db.connect() as conn: with conn.cursor( cursor_factory=psycopg2.extras.RealDictCursor) as cur: cur.execute(f"SELECT token FROM notify") fetchs = cur.fetchall() for fetch in fetchs: body = { 'token': f"{fetch['token']}", 'message': f"Hello everyone, {msg}" } cli.send_message(QueueUrl=os.getenv("SQS_URL"), DelaySeconds=0, MessageAttributes={}, MessageBody=json.dumps(body)) return {'result': 'ok'}, 200
def __init__(self, *args, **kwargs): """ Initialise attributes and register all behaviours """ self.logging = logging self.__log('Starting Assistant') self.db = Database() self.mode = kwargs.get('mode', 'console') self.behaviours = {} self.dir = os.path.dirname(os.path.realpath(__file__)) self.files = self.dir + '/files' self.config = config.Config() self.responder = None self.admin = self.config.get_or_request('Admin') self.register_behaviours() self.register_responders() schedule.clear() schedule.every(5).minutes.do(self.idle)
def getEvolutionData(players): #print (players[0]) retVal = dict() sortedByRating = sorted(players, key = lambda x : x[1],reverse=True)[:10] for player in sortedByRating: playerMap = dict() maptoAttach = dict() playerName = player[-2] x = player[0] temp = [] for row in Database().execute('select date,overall_rating from Player_Attributes where player_api_id is '+str(x)): temp.append(row) for row in temp: if playerMap.get(row[0].split('-')[0]) is None: playerMap[row[0].split('-')[0]] = [] playerMap[row[0].split('-')[0]].append(row[1]) #print(playerMap) retVal[playerName] = dict() for k,v in playerMap.items(): retVal[playerName][k]=np.mean(v) #print(retVal) data = [] featureVector = [] for player in retVal.keys(): row = [] featureVector.append(player) row.append(retVal.get(player).get('2007')) row.append(retVal.get(player).get('2008')) row.append(retVal.get(player).get('2009')) row.append(retVal.get(player).get('2010')) row.append(retVal.get(player).get('2011')) row.append(retVal.get(player).get('2012')) row.append(retVal.get(player).get('2013')) row.append(retVal.get(player).get('2014')) row.append(retVal.get(player).get('2015')) data.append(row) data = np.asarray(data) utils.createFile(data.T,"topPlayerRating.csv",featureVector)
class TestDBPrune(unittest.TestCase): def setUp(self) -> None: self.db = Database(':memory:') self.db.full_db_path = ':memory:' self.db.connect() populate_db(self.db.cur) def tearDown(self) -> None: self.db.close_conn() def test_delete_right_query(self): self.db.prune(200, NOW) self.assertEqual([MOCK_DATA['queries'][0]], self.db.cur.execute('SELECT * FROM queries') .fetchall()) def test_delete_correct_binding(self): self.db.prune(200, NOW) self.assertEqual([MOCK_DATA['query2torrent'][0], MOCK_DATA['query2torrent'][1], MOCK_DATA['query2torrent'][2]], self.db.cur.execute('SELECT * FROM query2torrent') .fetchall()) def test_delete_correct_torrents(self): self.db.prune(200, NOW) self.assertEqual([MOCK_DATA['torrents'][0], MOCK_DATA['torrents'][1], MOCK_DATA['torrents'][4]], self.db.cur.execute('SELECT id, name FROM torrents') .fetchall()) def test_do_not_delete(self): self.db.prune(301, NOW) self.assertEqual(MOCK_DATA['queries'], self.db.cur.execute('SELECT * FROM queries') .fetchall())
parser = argparse.ArgumentParser( description="Test Internet and Powercycle w/ Pi Zero") parser.add_argument("--test", action="store_true", help="Run Tests") parser.add_argument("--view", action="store_true", help="Select * from status.db") args = parser.parse_args() if args.test: test: TestInternet = TestInternet("https://facebook.com") result: TestInternetResult = test.result.value db = Database() db.add(result) last_five: List[Tuple[int, str, str]] = db.get_last_five_minutes() db.close() from pprint import pprint pprint(last_five) check_failed = CheckFailed(last_five) if check_failed.result == TestInternetResult.FAILED: rebooter: PiZero = PiZero() rebooter.power_off() rebooter.power_on() if args.view: db = Database()
def post(self): # get X-Line-Signature header value signature = request.headers['X-Line-Signature'] body = request.get_data(as_text=True) event = json.loads(body) print(event) try: self.handler.handle(body, signature) except InvalidSignatureError: print( "Invalid signature. Please check your channel access token/channel secret.") abort(400) token = event['events'][0]['replyToken'] if token == "00000000000000000000000000000000": pass else: profile = self.line_bot_api.get_profile( event['events'][0]['source']['userId']) print(profile) state = f"Hello 👉 `{profile.display_name}` 👈" id = profile.user_id picture = profile.picture_url name = profile.display_name try: with Database() as db, db.connect() as conn, conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: cur.execute( f"INSERT INTO users(id, name, picture) VALUES ('{id}', '{name}', '{picture}')") except Exception as e: print(e) pass message = event['events'][0]['message']['text'] if message == "上一頁" or message == "下一頁": try: rich_menu_id = self.line_bot_api.get_rich_menu_id_of_user( id) except: # link default rich menu self.line_bot_api.link_rich_menu_to_user( id, "richmenu-269cc28b8e8497d76c2df062b274a2ce") if rich_menu_id == "richmenu-269cc28b8e8497d76c2df062b274a2ce": self.line_bot_api.link_rich_menu_to_user( id, "richmenu-e31be74ad7e577b4752ab70c9c2a3fba") else: self.line_bot_api.link_rich_menu_to_user( id, "richmenu-269cc28b8e8497d76c2df062b274a2ce") else: self.line_bot_api.reply_message(token, TextSendMessage( text=message)) # buttons_template_message = TemplateSendMessage( # alt_text='Buttons template', # template=ButtonsTemplate( # thumbnail_image_url=f'{picture}.jpg', # title='Menu', # text='Please select', # actions=[ # PostbackAction( # label='postback', # display_text='postback text', # data='action=buy&itemid=1' # ), # MessageAction( # label='message', # text='message text' # ), # URIAction( # label='uri', # uri='http://example.com/' # ) # ] # ) # ) # self.line_bot_api.reply_message(token, buttons_template_message) # LINE reply Location message # self.line_bot_api.reply_message(token, LocationSendMessage( # title='my location', # address='Tokyo', # latitude=35.65910807942215, # longitude=139.70372892916203 # )) # LINE reply Image and Text message # self.line_bot_api.reply_message(token, [TextSendMessage( # text=state), ImageSendMessage( # original_content_url=profile.picture_url, preview_image_url=profile.picture_url)]) return 'OK'
class App(): def __init__(self): self._lock = Lock() if APP_DB: self._db = Database(addr=APP_DB, domain=APP) else: self._db = Database(addr=localhost(), domain=APP) def _print(self, text): if PRINT: show_info(self, text) def _get_repo(self, package): ring = HashRing(REPOSITORY_SERVERS) server = ring.get_node(package) return server def _install(self, buf): dirname = tempfile.mkdtemp() try: src = os.path.join(dirname, 'app.zip') with open(src, 'wb') as f: f.write(buf) dest = os.path.join(dirname, 'app') unzip_file(src, dest) cmd = 'python %s %s' % (PATH_INSTALLER, dest) status, output = commands.getstatusoutput(cmd) if status == 0: return output finally: shutil.rmtree(dirname) def install(self, uid, package, version): self._lock.acquire() try: if self._db.has_package(uid, package, None): show_error( self, 'failed to install, cannot install %s again' % package) return else: addr = self._get_repo(package) rpcclient = RPCClient(addr, REPOSITORY_PORT) if not version: version = rpcclient.request('version', package=package) if not version: show_error( self, 'failed to install, invalid version, uid=%s, package=%s' % (uid, package)) return ret = rpcclient.request('download', package=package, version=version) if ret: result = self._install(ret) if result: self._db.set_package(uid, package, version, result) self._print('finished installing %s, version=%s' % (package, version)) return True show_error(self, 'failed to install %s' % package) return finally: self._lock.release() def _uninstall(self, uid, package, info): pass def uninstall(self, uid, package): self._lock.acquire() try: if not self._db.has_package(uid, package, None): show_error(self, 'failed to uninstall %s ' % package) return version, info = self._db.get_package(uid, package, None) if info: self._uninstall(uid, package, info) self._db.rm_package(uid, package, version) return True finally: self._lock.release() def get_packages(self, uid): return self._db.get_packages(uid) def has_package(self, uid, package): return self._db.has_package(uid, package)
def __init__(self): self._lock = Lock() if APP_DB: self._db = Database(addr=APP_DB, domain=APP) else: self._db = Database(addr=localhost(), domain=APP)
def __init__(self) -> None: """Initialize the controller""" self.db = Database()
class Controller: """Class to control the dictdb""" def __init__(self) -> None: """Initialize the controller""" self.db = Database() def interact(self, args: Namespace) -> None: """start the interaction with the database""" if args.add: self.add(args.add) if args.search: self.search(args.search) if args.interactive: self.interactive() if args.list: self.list() if args.edit: self.edit(args.edit) if args.delete: self.delete(args.delete) def interactive(self) -> None: """ Launch editor with all the entries """ # load all the entries into editor entries = self.db.all() lines = [] for line in entries: entry = self.dict2entry(line) lines.append(entry.get_str()) raw = editor.edit(contents="\n".join(lines)) # update all the entries from editor to db lines = [line.strip("\n") for line in raw.decode().strip().split("\n")] # purge db and overwrite with current copy self.db.purge() for line in lines: entry = Entry(line) if entry.is_valid(): self.db.insert(entry.get_dict()) def load_content(self, content: [str]) -> None: """load the given content""" self.db.purge() if len(content) == 0: print("[-] Nothing to load") return print(f"[+] Loaded {len(content)} lines") imported = 0 for line in content: entry = Entry(line) if entry.is_valid(): self.db.insert(entry.get_dict()) imported += 1 print(f"[+] Imported {imported} entries") print(f"[+] Dictionary size : {self.db.count()}") def list(self) -> None: """list all the entries in the database""" words = self.db.all() if not words: print("[-] No words found in the database") return entries = [self.dict2entry(word) for word in words] for entry in entries: print(entry.get_str()) def add(self, data: str) -> None: entry = Entry(data) if not entry.is_valid(): # means parsing failed print("--add requires the data in the following format") print("--add 'word : (type) definition'") exit() self.db.insert(entry.get_dict()) def search(self, data: str) -> None: results = self.db.query(data) if not results: print("[-] Unable to find word in db") return for result in results: entry = self.dict2entry(result) print(entry.get_str()) def delete(self, data: str) -> None: """Function to delete the entry""" results = self.db.query(data) if not results: print("Unable to find word in db") return return self.db.delete(data) def edit(self, data: str) -> None: """ Function to modify the existing definitions If there are multiple word matches, all of them will be present in the editor If there is no match, empty editor will launch, and allow you to save new entry """ original_words = [] content = [] for word in self.db.query(data): entry = self.dict2entry(word) original_words.append(entry.word) content.append(entry.get_str()) # Launch editor with the matched entries raw = editor.edit(contents="\n".join(content)) # acquire the new content from the editor content = raw.decode().strip().split("\n") new_words = [] # Update existing entries or insert new entry for line in content: entry = Entry(line) if not entry.is_valid(): print(f"[-] Unable to parse the line: '{line}'") print("[-] word : (type) definition") else: new_words.append(entry.word) self.db.update(entry.get_dict()) # Check for deletions if len(new_words) != len(original_words): diff = set(original_words) - set(new_words) for entry in diff: self.db.delete(entry) def dict2entry(self, data: dict) -> Optional[Entry]: word = data.get("word") definition = data.get("definition") if word and definition: return Entry(f"{word} : {definition}") return None def entry2dict(self, entry: Entry) -> Optional[Dict]: if entry.is_valid(): return entry.get_dict() return None
def find(self,key: str) -> str: db = Database() code = db.find(key) if code: return Code(code) raise CodeNotFound('Code element not found.')
def validate_config(self): """Validate config""" dbase = Database() dbase.validate_config()
class Repository(RPCServer): def _print(self, text): if PRINT: show_info(self, text) def __init__(self, addr, port): RPCServer.__init__(self, addr, port) len_up = len(UPLOAD_SERVERS) len_repo = len(REPOSITORY_SERVERS) if len_up < len_repo or len_up % len_repo != 0: show_error(self, 'failed to initialize') raise Exception('failed to initialize') addr = localhost() if addr not in REPOSITORY_SERVERS: show_error(self, 'failed to initialize') raise Exception('failed to initialize REPOSITORY_SERVERS') for i in range(len(REPOSITORY_SERVERS)): if addr == REPOSITORY_SERVERS[i]: break total = len_up / len_repo self._upload_servers = UPLOAD_SERVERS[i * total:(i + 1) * total] self._print('upload_servers=%s' % str(self._upload_servers)) if HDFS: self._port = HDFS_PORT self._client = HDFSClient() else: self._port = FTP_PORT self._client = FTPClient() self._server = FTPServer() if REPO_DB: self._db = Database(addr=REPO_DB) else: self._db = Database(addr=addr) locks = [] for _ in range(LOCK_MAX): locks.append(Lock()) self._locks = HashRing(locks) if DEBUG: self._upload_cnt = 0 self._download_cnt = 0 def _get_addr(self, package): ring = HashRing(self._upload_servers) addr = ring.get_node(package) return addr def _get_lock(self, package): return self._locks.get_node(package) def _upload(self, package, version, buf): addr = self._get_addr(package) return self._client.upload(addr, self._port, package, version, buf) def upload(self, uid, package, version, buf): self._print('start to upload, uid=%s, package=%s, version=%s' % (str(uid), str(package), str(version))) lock = self._get_lock(package) lock.acquire() try: if SHOW_TIME: start_time = datetime.utcnow() owner, ver = self._db.get_version(package) if owner and owner != uid: show_error(self, 'failed to upload, invalid owner, package=%s, version=%s' % (str(package), str(version))) return False if ver == version:#The version of package has been uploaded. show_error(self, 'failed to upload, invalid version, package=%s, version=%s' % (str(package), str(version))) return False else: ret = self._upload(package, version, buf) if not ret: show_error(self, 'failed to upload, %s cannot upload %s-%s to hdfs' % (str(uid), str(package), str(version))) return False self._db.set_package(uid, package, version, '') if not ver or ver < version: self._db.set_version(uid, package, version) self._print('finished uploading, package=%s, version=%s' % (str(package), str(version))) if DEBUG: self._upload_cnt += 1 self._print('upload, count=%d' % self._upload_cnt) if SHOW_TIME: self._print('upload, time=%d sec' % (datetime.utcnow() - start_time).seconds) return True finally: lock.release() def download(self, package, version): self._print('start to download, package=%s, version=%s' % (str(package), str(version))) try: if SHOW_TIME: start_time = datetime.utcnow() addr = self._get_addr(package) uid, ver = self._db.get_version(package) if not version: version = ver if not self._db.has_package(uid, package, version): show_error(self, 'failed to download, invalid version, package=%s, version=%s' % (str(package), str(version))) return if DEBUG: self._download_cnt += 1 self._print('download, count=%d' % self._download_cnt) if SHOW_TIME: self._print( 'download, time=%d sec' % (datetime.utcnow() - start_time).seconds) return self._client.download(addr, self._port, package, version) except: show_error(self, 'failed to download') def version(self, package): self._print('start to get version, uid=%s, package=%s' % (str(uid), str(package))) _, ver = self._db.get_version(package) return ver def start(self): t = Thread(target=self.run) t.start() if not HDFS: self._server.run() t.join()
def setUp(self) -> None: self.db = Database(':memory:') self.db.full_db_path = ':memory:' self.db.connect() populate_db(self.db.cur)
class App(): def __init__(self): self._lock = Lock() if SERVER_APPDB: self._db = Database(addr=SERVER_APPDB[0], domain=APP) else: self._db = Database(addr=localhost(), domain=APP) def _print(self, text): if LOG_APP: show_info(self, text) def _get_repo(self, package): ring = HashRing(SERVER_REPO) server = ring.get_node(package) return server def _install(self, uid, content): if content: return True # output = installer.install(uid, content) # if output: # return output else: return str(None) def install(self, uid, package, version, content): self._lock.acquire() try: if self._db.has_package(uid, package, None): show_error( self, 'failed to install, cannot install %s again' % package) return else: result = self._install(uid, content) if result: self._db.set_package(uid, package, version, result) self._print('finished installing %s, version=%s' % (package, version)) return True finally: self._lock.release() def _uninstall(self, uid, package, info): pass def uninstall(self, uid, package): self._lock.acquire() try: if not self._db.has_package(uid, package, None): show_error(self, 'failed to uninstall %s' % package) return version, info = self._db.get_package(uid, package, None) if info: self._uninstall(uid, package, info) self._db.rm_package(uid, package, version) return True finally: self._lock.release() def get_packages(self, uid): return self._db.get_packages(uid) def has_package(self, uid, package): return self._db.has_package(uid, package)
from lib.db import Database if __name__ == "__main__": db = Database() users = {'micha': None, 'luke': None, 'steven': None} for user in users: print("Inserting user: "******"@example.com") users[user] = uid db.new_conversation(users['micha'], users['luke']) db.new_message(users['luke'], users['micha'], "Hi micha... this is luke") db.deanonymize_user(users['luke']) db.new_conversation(users['micha'], users['steven']) db.new_message(users['micha'], users['steven'], "hey steven... how are you?") db.new_message(users['steven'], users['micha'], "good! what about you?")
def all(self, private=False): db = Database() a = list() for row in db.all(private): a.append(Code(list(row))) return a
if key == "http" or key == "https" : # crawl only http protocol for url in urls[key]: tld = parser.tldExtractor(url) # crawl only onion sites if tld == "onion": retUrls.append(url) return urloc, retUrls except Exception as e: return urloc,[] if __name__ == "__main__": db = Database(PATH) urls = [WIKI,URLLISTING,OLDWIKI] while len(urls)>0: # multi threading function TPE default max workers == cpu count * 5 with concurrent.futures.ThreadPoolExecutor() as executor: # optimally defined number of threads urls = [executor.submit(crawl, url) for url in urls] concurrent.futures.wait(urls) newUrls = [] for result in urls: try: data = result.result()
def __init__(self): self._lock = Lock() if SERVER_APPDB: self._db = Database(addr=SERVER_APPDB[0], domain=APP) else: self._db = Database(addr=localhost(), domain=APP)
class App(): def __init__(self): self._lock = Lock() if APP_DB: self._db = Database(addr=APP_DB, domain=APP) else: self._db = Database(addr=localhost(), domain=APP) def _print(self, text): if PRINT: show_info(self, text) def _get_repo(self, package): ring = HashRing(REPOSITORY_SERVERS) server = ring.get_node(package) return server def _install(self, buf): dirname = tempfile.mkdtemp() try: src = os.path.join(dirname, 'app.zip') with open(src, 'wb') as f: f.write(buf) dest = os.path.join(dirname, 'app') unzip_file(src, dest) cmd = 'python %s %s' % (PATH_INSTALLER, dest) status, output = commands.getstatusoutput(cmd) if status == 0: return output finally: shutil.rmtree(dirname) def install(self, uid, package, version): self._lock.acquire() try: if self._db.has_package(uid, package, None): show_error(self, 'failed to install, cannot install %s again' % package) return else: addr = self._get_repo(package) rpcclient = RPCClient(addr, REPOSITORY_PORT) if not version: version = rpcclient.request('version', package=package) if not version: show_error(self, 'failed to install, invalid version, uid=%s, package=%s' % (uid, package)) return ret = rpcclient.request('download', package=package, version=version) if ret: result = self._install(ret) if result: self._db.set_package(uid, package, version, result) self._print('finished installing %s, version=%s' % (package, version)) return True show_error(self, 'failed to install %s' % package) return finally: self._lock.release() def _uninstall(self, uid, package, info): pass def uninstall(self, uid, package): self._lock.acquire() try: if not self._db.has_package(uid, package, None): show_error(self, 'failed to uninstall %s ' % package) return version, info = self._db.get_package(uid, package, None) if info: self._uninstall(uid, package, info) self._db.rm_package(uid, package, version) return True finally: self._lock.release() def get_packages(self, uid): return self._db.get_packages(uid) def has_package(self, uid, package): return self._db.has_package(uid, package)
def new(self, code, private, keylen=10): db = Database() return db.add(code, private, keylen)
class Repository(RPCServer): def _print(self, text): if PRINT: show_info(self, text) def __init__(self, addr, port): RPCServer.__init__(self, addr, port) len_up = len(UPLOAD_SERVERS) len_repo = len(REPOSITORY_SERVERS) if len_up < len_repo or len_up % len_repo != 0: show_error(self, 'failed to initialize') raise Exception('failed to initialize') addr = localhost() if addr not in REPOSITORY_SERVERS: show_error(self, 'failed to initialize') raise Exception('failed to initialize REPOSITORY_SERVERS') for i in range(len(REPOSITORY_SERVERS)): if addr == REPOSITORY_SERVERS[i]: break total = len_up / len_repo self._upload_servers = UPLOAD_SERVERS[i * total:(i + 1) * total] self._print('upload_servers=%s' % str(self._upload_servers)) if HDFS: self._port = HDFS_PORT self._client = HDFSClient() else: self._port = FTP_PORT self._client = FTPClient() self._server = FTPServer() if REPO_DB: self._db = Database(addr=REPO_DB) else: self._db = Database(addr=addr) locks = [] for _ in range(LOCK_MAX): locks.append(Lock()) self._locks = HashRing(locks) if DEBUG: self._upload_cnt = 0 self._download_cnt = 0 def _get_addr(self, package): ring = HashRing(self._upload_servers) addr = ring.get_node(package) return addr def _get_lock(self, package): return self._locks.get_node(package) def _upload(self, package, version, buf): addr = self._get_addr(package) return self._client.upload(addr, self._port, package, version, buf) def upload(self, uid, package, version, buf): self._print('start to upload, uid=%s, package=%s, version=%s' % (str(uid), str(package), str(version))) lock = self._get_lock(package) lock.acquire() try: if SHOW_TIME: start_time = datetime.utcnow() owner, ver = self._db.get_version(package) if owner and owner != uid: show_error( self, 'failed to upload, invalid owner, package=%s, version=%s' % (str(package), str(version))) return False if ver == version: #The version of package has been uploaded. show_error( self, 'failed to upload, invalid version, package=%s, version=%s' % (str(package), str(version))) return False else: ret = self._upload(package, version, buf) if not ret: show_error( self, 'failed to upload, %s cannot upload %s-%s to hdfs' % (str(uid), str(package), str(version))) return False self._db.set_package(uid, package, version, '') if not ver or ver < version: self._db.set_version(uid, package, version) self._print('finished uploading, package=%s, version=%s' % (str(package), str(version))) if DEBUG: self._upload_cnt += 1 self._print('upload, count=%d' % self._upload_cnt) if SHOW_TIME: self._print('upload, time=%d sec' % (datetime.utcnow() - start_time).seconds) return True finally: lock.release() def download(self, package, version): self._print('start to download, package=%s, version=%s' % (str(package), str(version))) try: if SHOW_TIME: start_time = datetime.utcnow() addr = self._get_addr(package) uid, ver = self._db.get_version(package) if not version: version = ver if not self._db.has_package(uid, package, version): show_error( self, 'failed to download, invalid version, package=%s, version=%s' % (str(package), str(version))) return if DEBUG: self._download_cnt += 1 self._print('download, count=%d' % self._download_cnt) if SHOW_TIME: self._print('download, time=%d sec' % (datetime.utcnow() - start_time).seconds) return self._client.download(addr, self._port, package, version) except: show_error(self, 'failed to download') def version(self, package): self._print('start to get version, uid=%s, package=%s' % (str(uid), str(package))) _, ver = self._db.get_version(package) return ver def start(self): t = Thread(target=self.run) t.start() if not HDFS: self._server.run() t.join()
def crawl(urloc:str) -> (str,list): db = Database(PATH) parser = Parser() session = connect_to_tor() # select here to find if in db try: urlindb = db.isCrawled(urloc) if len(urlindb) > 0: # url already crawled del urlindb return urloc,[] except Exception as e: print(e) try: try: r = session.get(urloc,headers=TORBUNDLEHEADER,timeout=20) r.raise_for_status() except Exception as err: insert_data = { "protocol" : "Error", "url" : urloc, "data" : base64.b64encode(str(err).encode()), "lastvisit": int(time.time()), } try: db.insert(insert_data) except Exception as e: # if urloc in db dont crawl it again and return if "UNIQUE constraint failed" in str(e): # update val try: db.update(insert_data) except Exception as e: pass else: urls = parser.urlExtractor(urloc,r.text) protocol = urloc.split("://")[0] insert_data = { "protocol" : protocol, "url" : urloc, "data" : base64.b64encode(r.content), "lastvisit": int(time.time()), } try: db.insert(insert_data) except Exception as e: # if urloc in db dont crawl it again and return if "UNIQUE constraint failed" in str(e): # update val try: db.update(insert_data) except Exception as e: pass retUrls = [] for key, value in urls.items(): if key == "http" or key == "https" : # crawl only http protocol for url in urls[key]: tld = parser.tldExtractor(url) # crawl only onion sites if tld == "onion": retUrls.append(url) return urloc, retUrls except Exception as e: return urloc,[]
#!usr/bin/python2.6 # encoding: utf-8 from lib.db import Database db = Database('db/zombie.sqlite') # db.ImportSpeech('data/new/') # db.ImportAccount('data/account/sohu.csv')