Пример #1
0
 def __init__(self, addr, port):
     RPCServer.__init__(self, addr, port)
     len_up = len(UPLOAD_SERVERS)
     len_repo = len(REPOSITORY_SERVERS)
     if len_up < len_repo or len_up % len_repo != 0:
         show_error(self, 'failed to initialize')
         raise Exception('failed to initialize')
     addr = localhost()
     if addr not in REPOSITORY_SERVERS:
         show_error(self, 'failed to initialize')
         raise Exception('failed to initialize REPOSITORY_SERVERS')
     for i in range(len(REPOSITORY_SERVERS)):
         if  addr == REPOSITORY_SERVERS[i]:
             break
     total = len_up / len_repo 
     self._upload_servers = UPLOAD_SERVERS[i * total:(i + 1) * total]
     self._print('upload_servers=%s' % str(self._upload_servers))
     if HDFS:
         self._port = HDFS_PORT
         self._client = HDFSClient()
     else:
         self._port = FTP_PORT
         self._client = FTPClient()
         self._server = FTPServer()
     if REPO_DB:
         self._db = Database(addr=REPO_DB)
     else:
         self._db = Database(addr=addr)
     locks = []
     for _ in range(LOCK_MAX):
         locks.append(Lock())
     self._locks = HashRing(locks)
     if DEBUG:
         self._upload_cnt = 0
         self._download_cnt = 0
Пример #2
0
 def __init__(self, addr, port):
     RPCServer.__init__(self, addr, port)
     len_up = len(UPLOAD_SERVERS)
     len_repo = len(REPOSITORY_SERVERS)
     if len_up < len_repo or len_up % len_repo != 0:
         show_error(self, 'failed to initialize')
         raise Exception('failed to initialize')
     addr = localhost()
     if addr not in REPOSITORY_SERVERS:
         show_error(self, 'failed to initialize')
         raise Exception('failed to initialize REPOSITORY_SERVERS')
     for i in range(len(REPOSITORY_SERVERS)):
         if addr == REPOSITORY_SERVERS[i]:
             break
     total = len_up / len_repo
     self._upload_servers = UPLOAD_SERVERS[i * total:(i + 1) * total]
     self._print('upload_servers=%s' % str(self._upload_servers))
     if HDFS:
         self._port = HDFS_PORT
         self._client = HDFSClient()
     else:
         self._port = FTP_PORT
         self._client = FTPClient()
         self._server = FTPServer()
     if REPO_DB:
         self._db = Database(addr=REPO_DB)
     else:
         self._db = Database(addr=addr)
     locks = []
     for _ in range(LOCK_MAX):
         locks.append(Lock())
     self._locks = HashRing(locks)
     if DEBUG:
         self._upload_cnt = 0
         self._download_cnt = 0
Пример #3
0
 def prepare(self, IP):
     self.db = Database(DBPATH)
     self.ip = IP
     self.zombie = sina_zombie(IP)
     zombie_info = self.db.get_zombie(0, 1, now())[0]
     self.upass = zombie_info[2].strip('\r').strip('\n')
     self.uname = zombie_info[1]
     self.uid = zombie_info[0]
Пример #4
0
 def post(self):
     parser = reqparse.RequestParser()
     parser.add_argument('code',
                         required=True,
                         help='code can not be blank!')
     args = parser.parse_args()
     code = args['code']
     # LINE notify json data
     client = {
         'grant_type': 'authorization_code',
         'code': code,
         'redirect_uri': os.getenv("NOTIFY_REDIRECT_URI"),
         'client_id': os.getenv("NOTIFY_CLIENT_ID"),
         'client_secret': os.getenv("NOTIFY_CLIENT_SECRET")
     }
     # send request to auth
     r = requests.post('https://notify-bot.line.me/oauth/token',
                       data=client)
     req = json.loads(r.text)
     if req['status'] == 200:
         token = req['access_token']
         # Here is use PostgreSQL, you can change your love db
         with Database() as db, db.connect() as conn:
             with conn.cursor(
                     cursor_factory=psycopg2.extras.RealDictCursor) as cur:
                 cur.execute(
                     f"INSERT INTO notify(token) VALUES ('{token}')")
         return {'access_token': req['access_token']}, 200
     else:
         return {'message': r.text}, 200
Пример #5
0
    def test_init(self, db):
        assert db is not None

        with pytest.raises(Exception) as exec_info:
            db = Database("/usr/bin/db.json")
        assert "not permitted" in str(
            exec_info.value) or "Permission denied" in str(exec_info.value)
Пример #6
0
	def prepare(self,IP):
		self.db = Database(DBPATH)
		self.ip = IP
		self.zombie = sina_zombie(IP)
		zombie_info = self.db.get_zombie(0,1,now())[0]
		self.upass = zombie_info[2].strip('\r').strip('\n')
		self.uname = zombie_info[1]
		self.uid = zombie_info[0]
Пример #7
0
 def get(self):
     msg = request.args.get('msg')
     with Database() as db, db.connect() as conn, conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
         cur.execute("SELECT * FROM users")
         fetch = cur.fetchall()
     for user in fetch:
         # LINE push message
         self.line_bot_api.push_message(
             user['id'], TextSendMessage(text=msg))
     return {'message': msg}, 200
Пример #8
0
def main():
    db = Database.get()
    db.create_tables([SynthButtonSetting, WavFile])

    drivers = load_drivers(config.SAMPLE_RATE)
    with AudioManager(drivers) as audio_manager:
        app = generate_flask_app(audio_manager)
        http_server = WSGIServer(("", 3001), app)
        signal.signal(signal.SIGINT,
                      lambda *args, **kwargs: http_server.stop())
        http_server.serve_forever()
Пример #9
0
def run_parsing(config):
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()
    # Search parser group
    search_parser = subparsers.add_parser('search')
    search_parser.add_argument('QUERY',
                               action='store',
                               type=str,
                               default=False)
    search_parser.add_argument('--output',
                               '-o',
                               action='store',
                               required=False,
                               default='json',
                               choices=['tsv', 'json', 'binary'])
    search_parser.add_argument('--columns',
                               '-c',
                               action='store',
                               required=False,
                               help='Coma separated names of columns')

    # Prune parser group
    prune_parser = subparsers.add_parser('prune')
    prune_parser.add_argument('PRUNE', action='store', type=int, default=False)

    args = parser.parse_args()

    try:
        db_path = os.environ['DB_PATH']
    except KeyError:
        print("Env variable DB_PATH is missing!", file=sys.stderr)
        exit(1)

    db = Database(db_path)
    if 'QUERY' in args:
        try:
            # Scrape and download
            down = Downloader()
            parser = Parser(config)
            d = Driver(config, args.QUERY, down, parser, db)
            torrents, headers = d.run()

            # Format and print
            fm = lib.utils.Formatter(headers, torrents)
            if args.columns:
                fm.filter_data(args.columns.split(','))
            fm.format_data(args.output)
            fm.print_data()
        except:
            traceback.print_exc()
            exit(1)

    elif 'PRUNE' in args:
        try:
            # Prune DB
            db.connect()
            db.prune(args.PRUNE, time.time())
        except:
            traceback.print_exc()
            exit(1)
Пример #10
0
class Config(object):

    def __init__(self, **kwargs):
        super(self.__class__, self).__init__(**kwargs)
        self.db = Database()

    def set(self, key, value):
        results = self.db.find_one('config', {'key': key})
        if results is not None:
            self.db.delete(results)

        self.db.insert('config', {'key': key, 'value': value})
        return 'Config set: ' + key + ' = ' + value

    def get(self, key):
        results = self.db.find_one('config', {'key': key})
        if results is None:
            raise ValueError(key + ' is not set in config')
            return None
        return results['value']

    def request(self, key):  # pragma: no cover
        if sys.version_info < (3, 0):
            value = raw_input("Enter value for config '" + key + "': ")
        else:
            value = input("Enter value for config '" + key + "': ")
        self.set(key, value)
        return value

    def get_or_request(self, key):
        try:
            value = self.get(key)
        except ValueError:
            value = self.request(key)
        return value
Пример #11
0
class Thread_Zombie(threading.Thread):
	"""docstring for ClassName"""
	def prepare(self,IP):
		self.db = Database(DBPATH)
		self.ip = IP
		self.zombie = sina_zombie(IP)
		zombie_info = self.db.get_zombie(0,1,now())[0]
		self.upass = zombie_info[2].strip('\r').strip('\n')
		self.uname = zombie_info[1]
		self.uid = zombie_info[0]
	def run(self):
		zombie_routine(self.zombie,self.uname,self.upass,self.uid,self.db)
		print self.ip,self.uname,self.upass,self.uid
Пример #12
0
 def __init__(self, **kwargs):
     self.act = {}
     self.db = kwargs.get('db', Database())
     self.dir = kwargs.get('dir', None)
     self.files = self.dir + '/files'
     self.config = kwargs.get('config', None)
     self.collection = ''
     self.match = None
     self.execution_order = 1
     self.logging = kwargs.get('logging', None)
     self.history = {}
     self.idle_methods = []
     self.define_idle(self.export_db, 24,
                      lib.dt.datetime_from_time(0, 0))  # export db nightly
Пример #13
0
class Thread_Zombie(threading.Thread):
    """docstring for ClassName"""
    def prepare(self, IP):
        self.db = Database(DBPATH)
        self.ip = IP
        self.zombie = sina_zombie(IP)
        zombie_info = self.db.get_zombie(0, 1, now())[0]
        self.upass = zombie_info[2].strip('\r').strip('\n')
        self.uname = zombie_info[1]
        self.uid = zombie_info[0]

    def run(self):
        zombie_routine(self.zombie, self.uname, self.upass, self.uid, self.db)
        print self.ip, self.uname, self.upass, self.uid
Пример #14
0
    def get(self):
        msg = request.args.get('msg')
        with Database() as db, db.connect() as conn:
            with conn.cursor(
                    cursor_factory=psycopg2.extras.RealDictCursor) as cur:
                cur.execute(f"SELECT token FROM notify")
                fetch = cur.fetchall()
        for f in fetch:
            headers = {
                'Content-Type': 'application/x-www-form-urlencoded',
                'Authorization': f"Bearer {f['token']}"
            }
            payload = {'message': msg}

            r = requests.post('https://notify-api.line.me/api/notify',
                              data=payload,
                              headers=headers)
        return {'result': 'ok'}, 200
Пример #15
0
 def post(self):
     parser = reqparse.RequestParser()
     parser.add_argument('message',
                         required=True,
                         help='message can not be blank!')
     args = parser.parse_args()
     msg = args['message']
     with Database() as db, db.connect() as conn:
         with conn.cursor(
                 cursor_factory=psycopg2.extras.RealDictCursor) as cur:
             cur.execute(f"SELECT token FROM notify")
             fetchs = cur.fetchall()
     for fetch in fetchs:
         body = {
             'token': f"{fetch['token']}",
             'message': f"Hello everyone, {msg}"
         }
         cli.send_message(QueueUrl=os.getenv("SQS_URL"),
                          DelaySeconds=0,
                          MessageAttributes={},
                          MessageBody=json.dumps(body))
     return {'result': 'ok'}, 200
Пример #16
0
    def __init__(self, *args, **kwargs):
        """ Initialise attributes and register all behaviours """
        self.logging = logging
        self.__log('Starting Assistant')
        self.db = Database()
        self.mode = kwargs.get('mode', 'console')

        self.behaviours = {}

        self.dir = os.path.dirname(os.path.realpath(__file__))
        self.files = self.dir + '/files'

        self.config = config.Config()
        self.responder = None

        self.admin = self.config.get_or_request('Admin')

        self.register_behaviours()
        self.register_responders()

        schedule.clear()
        schedule.every(5).minutes.do(self.idle)
Пример #17
0
def getEvolutionData(players):
    #print (players[0])
    retVal = dict()
    sortedByRating = sorted(players, key = lambda x  : x[1],reverse=True)[:10]
    for player in sortedByRating:
        playerMap = dict()
        maptoAttach = dict()
        playerName = player[-2]
        x = player[0]
        temp = []
        for row in Database().execute('select date,overall_rating from Player_Attributes where player_api_id is '+str(x)):
            temp.append(row)
        for row in temp:
            if playerMap.get(row[0].split('-')[0]) is None:
                playerMap[row[0].split('-')[0]] = []
            playerMap[row[0].split('-')[0]].append(row[1])
        #print(playerMap)
        retVal[playerName] = dict()
        for k,v in playerMap.items():
            retVal[playerName][k]=np.mean(v)
        #print(retVal)
    data = []
    featureVector = []
    for player in retVal.keys():
        row = []
        featureVector.append(player)
        row.append(retVal.get(player).get('2007'))
        row.append(retVal.get(player).get('2008'))
        row.append(retVal.get(player).get('2009'))
        row.append(retVal.get(player).get('2010'))
        row.append(retVal.get(player).get('2011'))
        row.append(retVal.get(player).get('2012'))
        row.append(retVal.get(player).get('2013'))
        row.append(retVal.get(player).get('2014'))
        row.append(retVal.get(player).get('2015'))
        data.append(row)
    data = np.asarray(data)
    utils.createFile(data.T,"topPlayerRating.csv",featureVector)
Пример #18
0
class TestDBPrune(unittest.TestCase):

    def setUp(self) -> None:
        self.db = Database(':memory:')
        self.db.full_db_path = ':memory:'
        self.db.connect()
        populate_db(self.db.cur)

    def tearDown(self) -> None:
        self.db.close_conn()

    def test_delete_right_query(self):
        self.db.prune(200, NOW)
        self.assertEqual([MOCK_DATA['queries'][0]],
                         self.db.cur.execute('SELECT * FROM queries')
                         .fetchall())

    def test_delete_correct_binding(self):
        self.db.prune(200, NOW)
        self.assertEqual([MOCK_DATA['query2torrent'][0],
                          MOCK_DATA['query2torrent'][1],
                          MOCK_DATA['query2torrent'][2]],
                         self.db.cur.execute('SELECT * FROM query2torrent')
                         .fetchall())

    def test_delete_correct_torrents(self):
        self.db.prune(200, NOW)
        self.assertEqual([MOCK_DATA['torrents'][0],
                          MOCK_DATA['torrents'][1],
                          MOCK_DATA['torrents'][4]],
                         self.db.cur.execute('SELECT id, name FROM torrents')
                         .fetchall())

    def test_do_not_delete(self):
        self.db.prune(301, NOW)
        self.assertEqual(MOCK_DATA['queries'],
                         self.db.cur.execute('SELECT * FROM queries')
                         .fetchall())
Пример #19
0
parser = argparse.ArgumentParser(
    description="Test Internet and Powercycle w/ Pi Zero")

parser.add_argument("--test", action="store_true", help="Run Tests")
parser.add_argument("--view",
                    action="store_true",
                    help="Select * from status.db")

args = parser.parse_args()

if args.test:
    test: TestInternet = TestInternet("https://facebook.com")
    result: TestInternetResult = test.result.value

    db = Database()
    db.add(result)
    last_five: List[Tuple[int, str, str]] = db.get_last_five_minutes()
    db.close()

    from pprint import pprint
    pprint(last_five)

    check_failed = CheckFailed(last_five)
    if check_failed.result == TestInternetResult.FAILED:
        rebooter: PiZero = PiZero()
        rebooter.power_off()
        rebooter.power_on()

if args.view:
    db = Database()
Пример #20
0
    def post(self):
        # get X-Line-Signature header value
        signature = request.headers['X-Line-Signature']

        body = request.get_data(as_text=True)
        event = json.loads(body)
        print(event)
        try:
            self.handler.handle(body, signature)
        except InvalidSignatureError:
            print(
                "Invalid signature. Please check your channel access token/channel secret.")
            abort(400)

        token = event['events'][0]['replyToken']
        if token == "00000000000000000000000000000000":
            pass
        else:
            profile = self.line_bot_api.get_profile(
                event['events'][0]['source']['userId'])
            print(profile)
            state = f"Hello 👉 `{profile.display_name}` 👈"
            id = profile.user_id
            picture = profile.picture_url
            name = profile.display_name
            try:
                with Database() as db, db.connect() as conn, conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
                    cur.execute(
                        f"INSERT INTO users(id, name, picture) VALUES ('{id}', '{name}', '{picture}')")
            except Exception as e:
                print(e)
                pass
            message = event['events'][0]['message']['text']
            if message == "上一頁" or message == "下一頁":
                try:
                    rich_menu_id = self.line_bot_api.get_rich_menu_id_of_user(
                        id)
                except:
                    # link default rich menu
                    self.line_bot_api.link_rich_menu_to_user(
                        id, "richmenu-269cc28b8e8497d76c2df062b274a2ce")
                if rich_menu_id == "richmenu-269cc28b8e8497d76c2df062b274a2ce":
                    self.line_bot_api.link_rich_menu_to_user(
                        id, "richmenu-e31be74ad7e577b4752ab70c9c2a3fba")
                else:
                    self.line_bot_api.link_rich_menu_to_user(
                        id, "richmenu-269cc28b8e8497d76c2df062b274a2ce")
            else:
                self.line_bot_api.reply_message(token, TextSendMessage(
                    text=message))
            # buttons_template_message = TemplateSendMessage(
            #     alt_text='Buttons template',
            #     template=ButtonsTemplate(
            #         thumbnail_image_url=f'{picture}.jpg',
            #         title='Menu',
            #         text='Please select',
            #         actions=[
            #             PostbackAction(
            #                 label='postback',
            #                 display_text='postback text',
            #                 data='action=buy&itemid=1'
            #             ),
            #             MessageAction(
            #                 label='message',
            #                 text='message text'
            #             ),
            #             URIAction(
            #                 label='uri',
            #                 uri='http://example.com/'
            #             )
            #         ]
            #     )
            # )
            # self.line_bot_api.reply_message(token, buttons_template_message)

            # LINE reply Location message
            # self.line_bot_api.reply_message(token, LocationSendMessage(
            #     title='my location',
            #     address='Tokyo',
            #     latitude=35.65910807942215,
            #     longitude=139.70372892916203
            # ))

            # LINE reply Image and Text message
            # self.line_bot_api.reply_message(token, [TextSendMessage(
            #     text=state), ImageSendMessage(
            #     original_content_url=profile.picture_url, preview_image_url=profile.picture_url)])
        return 'OK'
Пример #21
0
Файл: app.py Проект: tx626/dpm
class App():
    def __init__(self):
        self._lock = Lock()
        if APP_DB:
            self._db = Database(addr=APP_DB, domain=APP)
        else:
            self._db = Database(addr=localhost(), domain=APP)

    def _print(self, text):
        if PRINT:
            show_info(self, text)

    def _get_repo(self, package):
        ring = HashRing(REPOSITORY_SERVERS)
        server = ring.get_node(package)
        return server

    def _install(self, buf):
        dirname = tempfile.mkdtemp()
        try:
            src = os.path.join(dirname, 'app.zip')
            with open(src, 'wb') as f:
                f.write(buf)
            dest = os.path.join(dirname, 'app')
            unzip_file(src, dest)
            cmd = 'python %s %s' % (PATH_INSTALLER, dest)
            status, output = commands.getstatusoutput(cmd)
            if status == 0:
                return output
        finally:
            shutil.rmtree(dirname)

    def install(self, uid, package, version):
        self._lock.acquire()
        try:
            if self._db.has_package(uid, package, None):
                show_error(
                    self,
                    'failed to install, cannot install %s again' % package)
                return
            else:
                addr = self._get_repo(package)
                rpcclient = RPCClient(addr, REPOSITORY_PORT)
                if not version:
                    version = rpcclient.request('version', package=package)
                    if not version:
                        show_error(
                            self,
                            'failed to install, invalid version, uid=%s, package=%s'
                            % (uid, package))
                        return
                ret = rpcclient.request('download',
                                        package=package,
                                        version=version)
                if ret:
                    result = self._install(ret)
                    if result:
                        self._db.set_package(uid, package, version, result)
                        self._print('finished installing %s, version=%s' %
                                    (package, version))
                        return True
            show_error(self, 'failed to install %s' % package)
            return
        finally:
            self._lock.release()

    def _uninstall(self, uid, package, info):
        pass

    def uninstall(self, uid, package):
        self._lock.acquire()
        try:
            if not self._db.has_package(uid, package, None):
                show_error(self, 'failed to uninstall %s ' % package)
                return
            version, info = self._db.get_package(uid, package, None)
            if info:
                self._uninstall(uid, package, info)
            self._db.rm_package(uid, package, version)
            return True
        finally:
            self._lock.release()

    def get_packages(self, uid):
        return self._db.get_packages(uid)

    def has_package(self, uid, package):
        return self._db.has_package(uid, package)
Пример #22
0
Файл: app.py Проект: tx626/dpm
 def __init__(self):
     self._lock = Lock()
     if APP_DB:
         self._db = Database(addr=APP_DB, domain=APP)
     else:
         self._db = Database(addr=localhost(), domain=APP)
Пример #23
0
 def __init__(self) -> None:
     """Initialize the controller"""
     self.db = Database()
Пример #24
0
class Controller:
    """Class to control the dictdb"""
    def __init__(self) -> None:
        """Initialize the controller"""
        self.db = Database()

    def interact(self, args: Namespace) -> None:
        """start the interaction with the database"""
        if args.add:
            self.add(args.add)

        if args.search:
            self.search(args.search)

        if args.interactive:
            self.interactive()

        if args.list:
            self.list()

        if args.edit:
            self.edit(args.edit)

        if args.delete:
            self.delete(args.delete)

    def interactive(self) -> None:
        """
        Launch editor with all the entries
        """
        # load all the entries into editor
        entries = self.db.all()
        lines = []
        for line in entries:
            entry = self.dict2entry(line)
            lines.append(entry.get_str())

        raw = editor.edit(contents="\n".join(lines))

        # update all the entries from editor to db
        lines = [line.strip("\n") for line in raw.decode().strip().split("\n")]
        # purge db and overwrite with current copy
        self.db.purge()

        for line in lines:
            entry = Entry(line)
            if entry.is_valid():
                self.db.insert(entry.get_dict())

    def load_content(self, content: [str]) -> None:
        """load the given content"""
        self.db.purge()
        if len(content) == 0:
            print("[-] Nothing to load")
            return

        print(f"[+] Loaded {len(content)} lines")

        imported = 0
        for line in content:
            entry = Entry(line)
            if entry.is_valid():
                self.db.insert(entry.get_dict())
                imported += 1

        print(f"[+] Imported {imported} entries")
        print(f"[+] Dictionary size : {self.db.count()}")

    def list(self) -> None:
        """list all the entries in the database"""
        words = self.db.all()
        if not words:
            print("[-] No words found in the database")
            return

        entries = [self.dict2entry(word) for word in words]
        for entry in entries:
            print(entry.get_str())

    def add(self, data: str) -> None:
        entry = Entry(data)
        if not entry.is_valid():
            # means parsing failed
            print("--add requires the data in the following format")
            print("--add 'word : (type) definition'")
            exit()
        self.db.insert(entry.get_dict())

    def search(self, data: str) -> None:
        results = self.db.query(data)

        if not results:
            print("[-] Unable to find word in db")
            return

        for result in results:
            entry = self.dict2entry(result)
            print(entry.get_str())

    def delete(self, data: str) -> None:
        """Function to delete the entry"""
        results = self.db.query(data)

        if not results:
            print("Unable to find word in db")
            return

        return self.db.delete(data)

    def edit(self, data: str) -> None:
        """
        Function to modify the existing definitions
        If there are multiple word matches, all of them will be present in the editor
        If there is no match, empty editor will launch, and allow you to save new entry
        """
        original_words = []
        content = []
        for word in self.db.query(data):
            entry = self.dict2entry(word)
            original_words.append(entry.word)
            content.append(entry.get_str())

        # Launch editor with the matched entries
        raw = editor.edit(contents="\n".join(content))

        # acquire the new content from the editor
        content = raw.decode().strip().split("\n")

        new_words = []

        # Update existing entries or insert new entry
        for line in content:
            entry = Entry(line)
            if not entry.is_valid():
                print(f"[-] Unable to parse the line: '{line}'")
                print("[-] word : (type) definition")
            else:
                new_words.append(entry.word)
                self.db.update(entry.get_dict())

        # Check for deletions
        if len(new_words) != len(original_words):
            diff = set(original_words) - set(new_words)
            for entry in diff:
                self.db.delete(entry)

    def dict2entry(self, data: dict) -> Optional[Entry]:
        word = data.get("word")
        definition = data.get("definition")

        if word and definition:
            return Entry(f"{word} : {definition}")

        return None

    def entry2dict(self, entry: Entry) -> Optional[Dict]:
        if entry.is_valid():
            return entry.get_dict()
        return None
Пример #25
0
 def find(self,key: str) -> str:
         db = Database()
         code = db.find(key)
         if code:
                 return Code(code)
         raise CodeNotFound('Code element not found.')
Пример #26
0
    def validate_config(self):
        """Validate config"""

        dbase = Database()
        dbase.validate_config()
Пример #27
0
class Repository(RPCServer):
    def _print(self, text):
        if PRINT:
            show_info(self, text)
    
    def __init__(self, addr, port):
        RPCServer.__init__(self, addr, port)
        len_up = len(UPLOAD_SERVERS)
        len_repo = len(REPOSITORY_SERVERS)
        if len_up < len_repo or len_up % len_repo != 0:
            show_error(self, 'failed to initialize')
            raise Exception('failed to initialize')
        addr = localhost()
        if addr not in REPOSITORY_SERVERS:
            show_error(self, 'failed to initialize')
            raise Exception('failed to initialize REPOSITORY_SERVERS')
        for i in range(len(REPOSITORY_SERVERS)):
            if  addr == REPOSITORY_SERVERS[i]:
                break
        total = len_up / len_repo 
        self._upload_servers = UPLOAD_SERVERS[i * total:(i + 1) * total]
        self._print('upload_servers=%s' % str(self._upload_servers))
        if HDFS:
            self._port = HDFS_PORT
            self._client = HDFSClient()
        else:
            self._port = FTP_PORT
            self._client = FTPClient()
            self._server = FTPServer()
        if REPO_DB:
            self._db = Database(addr=REPO_DB)
        else:
            self._db = Database(addr=addr)
        locks = []
        for _ in range(LOCK_MAX):
            locks.append(Lock())
        self._locks = HashRing(locks)
        if DEBUG:
            self._upload_cnt = 0
            self._download_cnt = 0
    
    def _get_addr(self, package):
        ring = HashRing(self._upload_servers)
        addr = ring.get_node(package)
        return addr
    
    def _get_lock(self, package):
        return self._locks.get_node(package)
    
    def _upload(self, package, version, buf):
        addr = self._get_addr(package)
        return self._client.upload(addr, self._port, package, version, buf)
    
    def upload(self, uid, package, version, buf):
        self._print('start to upload, uid=%s, package=%s, version=%s' % (str(uid), str(package), str(version)))
        lock = self._get_lock(package)
        lock.acquire()
        try:
            if SHOW_TIME:
                start_time = datetime.utcnow()
            owner, ver = self._db.get_version(package)
            if owner and owner != uid:
                show_error(self, 'failed to upload, invalid owner, package=%s, version=%s' % (str(package), str(version)))
                return False
            if ver == version:#The version of package has been uploaded.
                show_error(self, 'failed to upload, invalid version, package=%s, version=%s' % (str(package), str(version)))
                return False
            else:
                ret = self._upload(package, version, buf)
                if not ret:
                    show_error(self, 'failed to upload, %s cannot upload %s-%s to hdfs' % (str(uid), str(package), str(version)))
                    return False
                self._db.set_package(uid, package, version, '')
                if not ver or ver < version:
                    self._db.set_version(uid, package, version)
                self._print('finished uploading, package=%s, version=%s' % (str(package), str(version)))
                if DEBUG:
                    self._upload_cnt += 1
                    self._print('upload, count=%d' % self._upload_cnt)
                if SHOW_TIME:
                    self._print('upload, time=%d sec' % (datetime.utcnow() - start_time).seconds)
                return True
        finally:
            lock.release()
    
    def download(self, package, version):
        self._print('start to download, package=%s, version=%s' % (str(package), str(version)))
        try:
            if SHOW_TIME:
                start_time = datetime.utcnow()
            addr = self._get_addr(package)
            uid, ver = self._db.get_version(package)
            if not version:
                version = ver
            if not self._db.has_package(uid, package, version):
                show_error(self, 'failed to download, invalid version, package=%s, version=%s' % (str(package), str(version)))
                return
            if DEBUG:
                self._download_cnt += 1
                self._print('download, count=%d' % self._download_cnt)
            if SHOW_TIME:
                self._print( 'download, time=%d sec' % (datetime.utcnow() - start_time).seconds)            
            return self._client.download(addr, self._port, package, version)
        except:
            show_error(self, 'failed to download')
    
    def version(self, package):
        self._print('start to get version, uid=%s, package=%s' % (str(uid), str(package)))
        _, ver = self._db.get_version(package)
        return ver
    
    def start(self):
        t = Thread(target=self.run)
        t.start()
        if not HDFS:
            self._server.run()
        t.join()
Пример #28
0
 def setUp(self) -> None:
     self.db = Database(':memory:')
     self.db.full_db_path = ':memory:'
     self.db.connect()
     populate_db(self.db.cur)
Пример #29
0
class App():
    def __init__(self):
        self._lock = Lock()
        if SERVER_APPDB:
            self._db = Database(addr=SERVER_APPDB[0], domain=APP)
        else:
            self._db = Database(addr=localhost(), domain=APP)

    def _print(self, text):
        if LOG_APP:
            show_info(self, text)

    def _get_repo(self, package):
        ring = HashRing(SERVER_REPO)
        server = ring.get_node(package)
        return server

    def _install(self, uid, content):
        if content:
            return True
#         output = installer.install(uid, content)
#         if output:
#             return output
        else:
            return str(None)

    def install(self, uid, package, version, content):
        self._lock.acquire()
        try:
            if self._db.has_package(uid, package, None):
                show_error(
                    self,
                    'failed to install, cannot install %s again' % package)
                return
            else:
                result = self._install(uid, content)
                if result:
                    self._db.set_package(uid, package, version, result)
                    self._print('finished installing %s, version=%s' %
                                (package, version))
                    return True
        finally:
            self._lock.release()

    def _uninstall(self, uid, package, info):
        pass

    def uninstall(self, uid, package):
        self._lock.acquire()
        try:
            if not self._db.has_package(uid, package, None):
                show_error(self, 'failed to uninstall %s' % package)
                return
            version, info = self._db.get_package(uid, package, None)
            if info:
                self._uninstall(uid, package, info)
            self._db.rm_package(uid, package, version)
            return True
        finally:
            self._lock.release()

    def get_packages(self, uid):
        return self._db.get_packages(uid)

    def has_package(self, uid, package):
        return self._db.has_package(uid, package)
Пример #30
0
from lib.db import Database

if __name__ == "__main__":
    db = Database()
    users = {'micha': None, 'luke': None, 'steven': None}
    for user in users:
        print("Inserting user: "******"@example.com")
        users[user] = uid
    db.new_conversation(users['micha'], users['luke'])
    db.new_message(users['luke'], users['micha'], "Hi micha... this is luke")
    db.deanonymize_user(users['luke'])

    db.new_conversation(users['micha'], users['steven'])
    db.new_message(users['micha'], users['steven'],
                   "hey steven... how are you?")
    db.new_message(users['steven'], users['micha'], "good! what about you?")
Пример #31
0
 def all(self, private=False):
         db = Database()
         a = list()
         for row in db.all(private):
                 a.append(Code(list(row)))
         return a
Пример #32
0
                if key == "http" or key == "https" :
                    # crawl only http protocol
                    for url in urls[key]:
                        tld = parser.tldExtractor(url)
                        # crawl only onion sites
                        if tld == "onion":
                            retUrls.append(url)
            return urloc, retUrls

    except Exception as e:
        return urloc,[]


if __name__ == "__main__":
    
    db   = Database(PATH)
    urls = [WIKI,URLLISTING,OLDWIKI]
    
    while len(urls)>0:
        
        # multi threading function TPE default max workers == cpu count * 5
        with concurrent.futures.ThreadPoolExecutor() as executor: # optimally defined number of threads
            
            urls = [executor.submit(crawl, url) for url in urls]
            concurrent.futures.wait(urls)
        
        newUrls = []
        
        for result in urls:
            try:
                data = result.result()
Пример #33
0
 def __init__(self):
     self._lock = Lock()
     if SERVER_APPDB:
         self._db = Database(addr=SERVER_APPDB[0], domain=APP)
     else:
         self._db = Database(addr=localhost(), domain=APP)
Пример #34
0
Файл: app.py Проект: tx626/dpm
class App():
    def __init__(self):
        self._lock = Lock()
        if APP_DB:
            self._db = Database(addr=APP_DB, domain=APP)
        else:
            self._db = Database(addr=localhost(), domain=APP)
    
    def _print(self, text):
        if PRINT:
            show_info(self, text)
    
    def _get_repo(self, package):
        ring = HashRing(REPOSITORY_SERVERS)
        server = ring.get_node(package)
        return server
    
    def _install(self, buf):
        dirname = tempfile.mkdtemp()
        try:
            src = os.path.join(dirname, 'app.zip')
            with open(src, 'wb') as f:
                f.write(buf)
            dest = os.path.join(dirname, 'app')
            unzip_file(src, dest)
            cmd = 'python %s %s' % (PATH_INSTALLER, dest)
            status, output = commands.getstatusoutput(cmd)
            if status ==  0:
                return output
        finally:
            shutil.rmtree(dirname)
    
    def install(self, uid, package, version):
        self._lock.acquire()
        try:
            if self._db.has_package(uid, package, None):
                show_error(self, 'failed to install, cannot install %s again' % package)
                return
            else:
                addr = self._get_repo(package)
                rpcclient = RPCClient(addr, REPOSITORY_PORT)
                if not version:
                    version = rpcclient.request('version', package=package)
                    if not version:
                        show_error(self, 'failed to install, invalid version, uid=%s, package=%s' % (uid, package))
                        return
                ret = rpcclient.request('download', package=package, version=version)
                if ret:
                    result = self._install(ret)
                    if result:
                        self._db.set_package(uid, package, version, result)
                        self._print('finished installing %s, version=%s' % (package, version))
                        return True
            show_error(self, 'failed to install %s' % package)
            return
        finally:
            self._lock.release()
        
    def _uninstall(self, uid, package, info):
        pass
        
    def uninstall(self, uid, package):
        self._lock.acquire()
        try:
            if not self._db.has_package(uid, package, None):
                show_error(self, 'failed to uninstall %s ' % package)
                return
            version, info = self._db.get_package(uid, package, None)
            if info:
                self._uninstall(uid, package, info)
            self._db.rm_package(uid, package, version)
            return True
        finally:
            self._lock.release()
    
    def get_packages(self, uid):
        return self._db.get_packages(uid)
    
    def has_package(self, uid, package):
        return self._db.has_package(uid, package)
Пример #35
0
 def new(self, code, private, keylen=10):
         db = Database()
         return db.add(code, private, keylen)
Пример #36
0
class Repository(RPCServer):
    def _print(self, text):
        if PRINT:
            show_info(self, text)

    def __init__(self, addr, port):
        RPCServer.__init__(self, addr, port)
        len_up = len(UPLOAD_SERVERS)
        len_repo = len(REPOSITORY_SERVERS)
        if len_up < len_repo or len_up % len_repo != 0:
            show_error(self, 'failed to initialize')
            raise Exception('failed to initialize')
        addr = localhost()
        if addr not in REPOSITORY_SERVERS:
            show_error(self, 'failed to initialize')
            raise Exception('failed to initialize REPOSITORY_SERVERS')
        for i in range(len(REPOSITORY_SERVERS)):
            if addr == REPOSITORY_SERVERS[i]:
                break
        total = len_up / len_repo
        self._upload_servers = UPLOAD_SERVERS[i * total:(i + 1) * total]
        self._print('upload_servers=%s' % str(self._upload_servers))
        if HDFS:
            self._port = HDFS_PORT
            self._client = HDFSClient()
        else:
            self._port = FTP_PORT
            self._client = FTPClient()
            self._server = FTPServer()
        if REPO_DB:
            self._db = Database(addr=REPO_DB)
        else:
            self._db = Database(addr=addr)
        locks = []
        for _ in range(LOCK_MAX):
            locks.append(Lock())
        self._locks = HashRing(locks)
        if DEBUG:
            self._upload_cnt = 0
            self._download_cnt = 0

    def _get_addr(self, package):
        ring = HashRing(self._upload_servers)
        addr = ring.get_node(package)
        return addr

    def _get_lock(self, package):
        return self._locks.get_node(package)

    def _upload(self, package, version, buf):
        addr = self._get_addr(package)
        return self._client.upload(addr, self._port, package, version, buf)

    def upload(self, uid, package, version, buf):
        self._print('start to upload, uid=%s, package=%s, version=%s' %
                    (str(uid), str(package), str(version)))
        lock = self._get_lock(package)
        lock.acquire()
        try:
            if SHOW_TIME:
                start_time = datetime.utcnow()
            owner, ver = self._db.get_version(package)
            if owner and owner != uid:
                show_error(
                    self,
                    'failed to upload, invalid owner, package=%s, version=%s' %
                    (str(package), str(version)))
                return False
            if ver == version:  #The version of package has been uploaded.
                show_error(
                    self,
                    'failed to upload, invalid version, package=%s, version=%s'
                    % (str(package), str(version)))
                return False
            else:
                ret = self._upload(package, version, buf)
                if not ret:
                    show_error(
                        self,
                        'failed to upload, %s cannot upload %s-%s to hdfs' %
                        (str(uid), str(package), str(version)))
                    return False
                self._db.set_package(uid, package, version, '')
                if not ver or ver < version:
                    self._db.set_version(uid, package, version)
                self._print('finished uploading, package=%s, version=%s' %
                            (str(package), str(version)))
                if DEBUG:
                    self._upload_cnt += 1
                    self._print('upload, count=%d' % self._upload_cnt)
                if SHOW_TIME:
                    self._print('upload, time=%d sec' %
                                (datetime.utcnow() - start_time).seconds)
                return True
        finally:
            lock.release()

    def download(self, package, version):
        self._print('start to download, package=%s, version=%s' %
                    (str(package), str(version)))
        try:
            if SHOW_TIME:
                start_time = datetime.utcnow()
            addr = self._get_addr(package)
            uid, ver = self._db.get_version(package)
            if not version:
                version = ver
            if not self._db.has_package(uid, package, version):
                show_error(
                    self,
                    'failed to download, invalid version, package=%s, version=%s'
                    % (str(package), str(version)))
                return
            if DEBUG:
                self._download_cnt += 1
                self._print('download, count=%d' % self._download_cnt)
            if SHOW_TIME:
                self._print('download, time=%d sec' %
                            (datetime.utcnow() - start_time).seconds)
            return self._client.download(addr, self._port, package, version)
        except:
            show_error(self, 'failed to download')

    def version(self, package):
        self._print('start to get version, uid=%s, package=%s' %
                    (str(uid), str(package)))
        _, ver = self._db.get_version(package)
        return ver

    def start(self):
        t = Thread(target=self.run)
        t.start()
        if not HDFS:
            self._server.run()
        t.join()
Пример #37
0
Файл: app.py Проект: tx626/dpm
 def __init__(self):
     self._lock = Lock()
     if APP_DB:
         self._db = Database(addr=APP_DB, domain=APP)
     else:
         self._db = Database(addr=localhost(), domain=APP)
Пример #38
0
def crawl(urloc:str) -> (str,list):
    db      = Database(PATH)
    parser  = Parser()
    session = connect_to_tor()

    # select here to find if in db 
    try:
        urlindb = db.isCrawled(urloc)
        if len(urlindb) > 0:
            # url already crawled
            del urlindb
            return urloc,[]
    except Exception as e:
        print(e)


    try:
        try:
            r = session.get(urloc,headers=TORBUNDLEHEADER,timeout=20)
            r.raise_for_status()

        except Exception as err:
            insert_data = {
            "protocol" : "Error",
            "url"      : urloc,
            "data"     : base64.b64encode(str(err).encode()),
            "lastvisit": int(time.time()),
            }

            try:
                db.insert(insert_data)
            except Exception as e:
                # if urloc in db dont crawl it again and return
                if "UNIQUE constraint failed" in str(e):
                    # update val
                    try:
                        db.update(insert_data)
                    except Exception as e:
                        pass    
        else:
            urls        = parser.urlExtractor(urloc,r.text)
            protocol    = urloc.split("://")[0]
            insert_data = {
            "protocol" : protocol,
            "url"      : urloc,
            "data"     : base64.b64encode(r.content),
            "lastvisit": int(time.time()),
            }
            
            try:
                db.insert(insert_data)
            except Exception as e:
                # if urloc in db dont crawl it again and return
                if "UNIQUE constraint failed" in str(e):
                    # update val
                    try:
                        db.update(insert_data)
                    except Exception as e:
                        pass

            retUrls = []
            for key, value in urls.items():
                if key == "http" or key == "https" :
                    # crawl only http protocol
                    for url in urls[key]:
                        tld = parser.tldExtractor(url)
                        # crawl only onion sites
                        if tld == "onion":
                            retUrls.append(url)
            return urloc, retUrls

    except Exception as e:
        return urloc,[]
Пример #39
0
#!usr/bin/python2.6
# encoding: utf-8
from lib.db import Database

db = Database('db/zombie.sqlite')
# db.ImportSpeech('data/new/')
# db.ImportAccount('data/account/sohu.csv')