Exemple #1
0
    def _run_map(self):

        args = self.args
        # if doing anything online, revalidate cache
        Cacher.revalidate(args.map_id, self.users_info)

        if (args.map_id and args.user_id):  # passed both -m and -u but not -l
            replays2 = [
                OnlineReplay.from_map(args.map_id, user_id, args.cache,
                                      replay_id)
                for user_id, replay_id in self.users_info.items()
            ]
            comparer = Comparer(args.threshold,
                                self.replays_check,
                                replays2=replays2)
            comparer.compare(mode="double")
            return

        if (args.map_id):  # only passed -m
            # get all 50 top replays
            replays = [
                OnlineReplay.from_map(args.map_id, user_id, args.cache,
                                      replay_id)
                for user_id, replay_id in self.users_info.items()
            ]
            comparer = Comparer(args.threshold, replays)
            comparer.compare(mode="single")
            return
Exemple #2
0
    def __init__(self, args):
        """
        Initializes a Circleguard instance.

        [SimpleNamespace or argparse.Namespace] args:
            A namespace-like object representing how and what to compare. An example may look like
            `Namespace(cache=False, local=False, map_id=None, number=50, threshold=20, user_id=None)`
        """

        # get all replays in path to check against. Load this per circleguard instance or users moving files around while the gui is open doesn't work.
        self.PATH_REPLAYS = [
            join(PATH_REPLAYS_STUB, f) for f in os.listdir(PATH_REPLAYS_STUB)
            if isfile(join(PATH_REPLAYS_STUB, f)) and f != ".DS_Store"
        ]

        self.cacher = Cacher(args.cache)
        self.args = args
        if (args.map_id):
            self.users_info = Loader.users_info(args.map_id, args.number)
        if (args.user_id and args.map_id):
            user_info = Loader.user_info(
                args.map_id, args.user_id
            )[args.
              user_id]  # should be guaranteed to only be a single mapping of user_id to a list
            self.replays_check = [
                OnlineReplay.from_map(self.cacher, args.map_id, args.user_id,
                                      user_info[0], user_info[1], user_info[2])
            ]
Exemple #3
0
class WebStash:
    def __init__(self, getterType='urlopen', waitTimeBeforeScraping=0):
        self.cacher = Cacher()
        self.config = Config()
        self.getter = Getter(getterType,
                             waitTimeBeforeScraping=waitTimeBeforeScraping)

    def get_web_data(self, url):
        try:
            return self.cacher[url]
        except KeyError:
            self.config.debugPrint('Getting webData...')
            filename = self.cacher.getFilename(url)
            html = self.getter.get_html(url)
            screenshotLocation = self.getter.get_screenshot(
                url, filename + '.png')
            webData = WebData(filename,
                              url,
                              html,
                              screenshotLocation=screenshotLocation)
            self.cacher[url] = webData
            return self.cacher[url]

    def delete(url):
        del self.cacher[url]

    def clean(self):
        self.cacher.clean()
Exemple #4
0
async def download_file(url, filename, cacher: Cacher, useragent=''):
    print(url)
    async with aiohttp.request('get', url, headers={
            'user-agent': useragent,
    }) as res:
        content = await res.read()
        cacher.set(filename, content)
Exemple #5
0
 def __init__(self, reporter, waiter, outdir, useragent) -> None:
     super(Keiba, self).__init__()
     self.reporter: Reporter = reporter
     self.waiter = waiter
     self.outdir = outdir
     self.useragent = useragent
     self.cacher = Cacher(self.outdir)
     self.semaphore = Semaphore(2)
Exemple #6
0
 def enable_cache(self, host="localhost", port=27017, test=False):
     """
     @param host: the hostname of the server where mongodb is running
     @param port: port mongodb is listening on
     @param test: True if you want to work on a db separate than the main cache
     """
     self._cache_connection_host = host
     self._cache_connection_port = port
     self._cache_enabled = True
     self._cacher = Cacher(self._cache_connection_host,
                           self._cache_connection_port, test)
Exemple #7
0
class Anicobin(Collector):
    def __init__(self, reporter, waiter, outdir, useragent) -> None:
        super(Anicobin, self).__init__()
        self.reporter: Reporter = reporter
        self.waiter = waiter
        self.outdir = outdir
        self.useragent = useragent
        self.cacher = Cacher(self.outdir)
        self.semaphore = Semaphore(2)

    async def get(self, url):
        filename = urllib.parse.quote(url, safe='') + '.html'
        cache, _ = self.cacher.get(filename)
        if cache:
            html = cache
        else:
            await self.waiter.wait(url)
            print('fetching', url)
            async with aiohttp.request('get',
                                       url,
                                       headers={'user-agent':
                                                self.useragent}) as req:
                content = await req.read()
                html = content.decode(SITE_ENCODING)
                self.cacher.set(filename, html)

        return html

    async def collect(self, base_url, queue_size=3):
        async def f(page):
            print(page)
            html, _ = await self.async_retry(3, self.get,
                                             f'{base_url}?p={page}')
            result = []
            for post_url in await self.run_in_executor(get_post_urls, html):
                _html = await self.get(post_url)
                urls = get_pict_urls(_html)
                for url in urls:
                    filename = urllib.parse.quote(url, safe='')
                    content, _ = self.cacher.get(filename, binary=True)
                    if not content:
                        await self.add_future(
                            'dlimage', download_file(url, filename,
                                                     self.cacher))

                result.extend(urls)

            return len(result) > 0

        await self.queued_paging(1,
                                 1000,
                                 lambda page: f(page),
                                 queue_size=queue_size)
Exemple #8
0
    def __init__(self, wallet_config):
        self._errors = []
        self._config = {}
        self._cache = Cacher({
            'accounts': {},
            'transactions': {},
            'transactiondetails': {},
            'balances': {},
            'info': {},
        })

        if type(wallet_config) is dict:
            self._config = wallet_config
Exemple #9
0
    def __init__(self, accountDetails):
        self._errors = []
        self._account = {}
        self._hidden = False
        self._cache = Cacher({
            'transactions': {},
            'balances': {},
            'addressesbyaccount': {},
        })

        if type(accountDetails) is dict:
            self._account = accountDetails
            self._provider_id = accountDetails['provider_id']
Exemple #10
0
    def default_request( self, req ):
        target = 'z_robot_web_tools'
        m = re.match( '/jenkins/(.*)', req.path_info )
        if m:
            target = m.group( 1 )

        cacher = Cacher( cacheConfig )
        data = cacher.deliver( target, mapFnc=self.cacher_deliver_mapFnc )
        data[ 'a_refresh' ] = \
            html.A( 'Refresh now', href=req.href.jenkins( 'refresh' ))
        data[ 'tabs' ] = self._tab_data
        data[ 'target' ] = target
        data[ 'tabs_base' ] = req.base_path + '/jenkins'
        return 'jenkins.html', data, None
Exemple #11
0
 def __init__(self,
              reporter: Reporter,
              waiter: Waiter,
              outdir: str,
              useragent: str = ''):
     super(WearCollector, self).__init__()
     self.reporter: Reporter = reporter
     self.waiter = waiter
     self.outdir = outdir
     self.useragent = useragent
     self.cacher = Cacher(self.outdir)
     # 非同期処理の同時接続数制御
     self.semaphore = Semaphore(2)
     # ファイルダウンローダ
     self.downloader = Downloader(self.waiter, self.semaphore,
                                  self.reporter)
Exemple #12
0
    def __init__(self, args):
        """
        Initializes an Anticheat instance.

        [SimpleNamespace or argparse.Namespace] args:
            A namespace-like object representing how and what to compare. An example may look like
            `Namespace(cache=False, local=False, map_id=None, number=50, threshold=20, user_id=None)`
        """

        self.cacher = Cacher(args.cache)
        self.args = args
        if(args.map_id):
            self.users_info = Loader.users_info(args.map_id, args.number)
        if(args.user_id and args.map_id):
            user_info = Loader.user_info(args.map_id, args.user_id)
            self.replays_check = [OnlineReplay.from_map(self.cacher, args.map_id, args.user_id, user_info[args.user_id][0], user_info[args.user_id][1])]
Exemple #13
0
    def __init__(self, transactionDetails):
        self._transaction = {}
        self._cache = Cacher({
            'details': {},
        })

        if type(transactionDetails) is dict:
            self._transaction = transactionDetails

            self['timereceived_pretty'] = misc.twitterizeDate(
                self.get('timereceived', 'never'))
            self['time_pretty'] = misc.twitterizeDate(self.get(
                'time', 'never'))
            self['timereceived_human'] = datetime.datetime.fromtimestamp(
                self.get('timereceived', 0))
            self['time_human'] = datetime.datetime.fromtimestamp(
                self.get('time', 0))
            self['blocktime_human'] = datetime.datetime.fromtimestamp(
                self.get('blocktime', 0))
            self['blocktime_pretty'] = misc.twitterizeDate(
                self.get('blocktime', 'never'))
            self['currency_symbol'] = misc.getCurrencySymbol(
                connector, self['currency'])

            if self.get('category', False) in ['receive', 'send']:
                if self['confirmations'] <= MainConfig['globals'][
                        'confirmation_limit']:
                    self['status_icon'] = 'glyphicon-time'
                    self['status_color'] = '#AAA'
                    self['tooltip'] = self['confirmations']
                else:
                    self['status_icon'] = 'glyphicon-ok-circle'
                    self['status_color'] = '#1C9E3F'
                    self['tooltip'] = self['confirmations']

            accountObject = self['wallet'].getAccountByName(self['account'])
            self['account'] = accountObject

            if self['category'] == 'receive':
                self['icon'] = 'glyphicon-circle-arrow-down'
            elif self['category'] == 'send':
                self['icon'] = 'glyphicon-circle-arrow-up'
            elif self['category'] == 'move':
                self['icon'] = 'glyphicon-circle-arrow-right'
                self['otheraccount'] = self['wallet'].getAccountByName(
                    self['otheraccount'])
Exemple #14
0
 def wrapper(*args, **kwargs):
     map_id = args[0]
     user_id = args[1]
     lzma = Cacher.check_cache(map_id, user_id)
     if (lzma):
         replay_data = osrparse.parse_replay(lzma, pure_lzma=True).play_data
         return Replay(replay_data, user_id)
     else:
         return function(*args, **kwargs)
Exemple #15
0
    def from_map(map_id, user_id, cache, replay_id):
        """
        Creates a Replay instance from a replay by the given user on the given map.

        Args:
            String map_id: The map_id to download the replay from.
            String user_id: The user id to download the replay of.
                            Also used as the username of the Replay.

        Returns:
            The Replay instance created with the given information.
        """

        lzma_bytes = Loader.replay_data(map_id, user_id)
        parsed_replay = osrparse.parse_replay(lzma_bytes, pure_lzma=True)
        replay_data = parsed_replay.play_data
        if (cache):
            Cacher.cache(map_id, user_id, lzma_bytes, replay_id)
        return OnlineReplay(replay_data, user_id)
Exemple #16
0
 def enable_cache(self, host="localhost", port=27017, test=False):
     """
     @param host: the hostname of the server where mongodb is running
     @param port: port mongodb is listening on
     @param test: True if you want to work on a db separate than the main cache
     """
     self._cache_connection_host = host
     self._cache_connection_port = port
     self._cache_enabled = True
     self._cacher = Cacher(self._cache_connection_host, self._cache_connection_port, test)
Exemple #17
0
 def __init__(self, wallet_config):
     self._errors = []
     self._config = {}
     self._cache = Cacher({
          'accounts': {},
          'transactions': {},
          'transactiondetails': {},
          'balances': {},
          'info': {},
          })
     
     if type(wallet_config) is dict:
         self._config = wallet_config
Exemple #18
0
def setup():
    try:
        shutil.rmtree('webstashcache')
    except FileNotFoundError:
        pass

    try:
        os.remove('cacheMap.pkl')
    except FileNotFoundError:
        pass
    cacher = Cacher()
    link = 'https://news.ycombinator.com/news'
    return (cacher, link)
 def __init__(self, accountDetails):
     self._errors = []
     self._account = {}
     self._hidden = False
     self._cache = Cacher({
      'transactions': {},
      'balances': {},
      'addressesbyaccount': {},
      })
     
     if type(accountDetails) is dict:
         self._account = accountDetails
         self._provider_id = accountDetails['provider_id']
Exemple #20
0
def get_element_processor(element, send_cache, verbose, dryrun):
    """Gets an instance of element that contains cached data, depending on 
    arguments passed.

    Will skip any cached data if send_cache is False or dryrun is True.

    @param element: Element class that should be fetched.
    @type element: L{MetaElement} sub class.
    @param send_cache: Indicates whether cache should be included.
    @type send_cache: bool
    @param verbose: Indicates whether or not the script runs in verbose mode.
    @type verbose: bool
    @param dryrun: Indicates whether the script is doing a dry run.
    @type dryrun: bool
    @return: L{MetaElement} sub class. The sub class is given by the element 
            parameter.

    """
    if dryrun or not send_cache:
        return element()
    if send_cache:
        c = Cacher(element.xml_tag_name)
        cached_data = c.get_cache()
        if cached_data is not None:
            if element.resend_cache:
                element_processor = element.from_xml_element(cached_data, 
                                                                element)
            else:
                logging.info(("Found cached data for \"%s\", but element "
                    "type declares not to resend this cache. "
                    "Removing cache.") % element.xml_tag_name)
                c.remove_cache()
                element_processor = element()
            if element_processor is None:
                logging.error(("Found cached data for \"%s\", but unable to "
                    "load. Check file \"%s\" for possible errors. "
                    "Continuing without cached data") % 
                                (element.xml_tag_name, c.file_path))
                element_processor = element()
            else:
                # We have successfully loaded the cached data.
                if element.resend_cache:
                    logging.debug(("Succesfully loaded cache for \"%s\", "
                        "removing cached file \"%s\".") % 
                        (element.xml_tag_name, c.file_path))
                    c.remove_cache()
        else:
            logging.debug(("Found no cached data for \"%s\".") % 
                            element.xml_tag_name)
            element_processor = element()
    return element_processor
 def __init__(self, transactionDetails):
     self._transaction = {}
     self. _cache = Cacher({
                  'details': {},
                  })
     
     if type(transactionDetails) is dict:
         self._transaction = transactionDetails
         
         self['timereceived_pretty'] = misc.twitterizeDate(self.get('timereceived', 'never'))
         self['time_pretty'] = misc.twitterizeDate(self.get('time', 'never'))
         self['timereceived_human'] = datetime.datetime.fromtimestamp(self.get('timereceived', 0))
         self['time_human'] = datetime.datetime.fromtimestamp(self.get('time', 0))
         self['blocktime_human'] = datetime.datetime.fromtimestamp(self.get('blocktime', 0))
         self['blocktime_pretty'] = misc.twitterizeDate(self.get('blocktime', 'never'))
         self['currency_symbol'] = misc.getCurrencySymbol(connector, self['currency'])
         
         if self.get('category', False) in ['receive', 'send']:
             if self['confirmations'] <= MainConfig['globals']['confirmation_limit']:
                 self['status_icon'] = 'glyphicon-time'
                 self['status_color'] = '#AAA';
                 self['tooltip'] = self['confirmations']
             else:
                 self['status_icon'] = 'glyphicon-ok-circle'
                 self['status_color'] = '#1C9E3F';
                 self['tooltip'] = self['confirmations']
         
         accountObject = self['wallet'].getAccountByName(self['account'])
         self['account'] = accountObject
         
         if self['category'] == 'receive':
             self['icon'] = 'glyphicon-circle-arrow-down'
         elif self['category'] == 'send':
             self['icon'] = 'glyphicon-circle-arrow-up'
         elif self['category'] == 'move':
             self['icon'] = 'glyphicon-circle-arrow-right'
             self['otheraccount'] = self['wallet'].getAccountByName(self['otheraccount'])
Exemple #22
0
 def refresh_request( self, req ):
     cacher = Cacher( self._tab_data )
     cacher.cache()
     success = True
     return 'refresh.html', { 'success': success }, None
Exemple #23
0
class CoinTransaction(object):
    '''
    Class for a transaction
    '''

    # source: https://github.com/zamgo/PHPCoinAddress/blob/master/README.md
    prefixes = {
        'btc': {
            'mainnet': '\x00',
            'testnet': '\x6f'
        },
        'ltc': {
            'mainnet': '\x30',
            'testnet': '\x6f'
        },
        'ftc': {
            'mainnet': '\x0E',
            'testnet': '\x6f'
        },
        'ppc': {
            'mainnet': '\x37',
            'testnet': '\x6f'
        },
        'nmc': {
            'mainnet': '\x34',
            'testnet': '\x6f'
        },
        'nvc': {
            'mainnet': '\x08',
            'testnet': '\x6f'
        },
        'doge': {
            'mainnet': '\x30',
            'testnet': '\x6f'
        },
    }

    def __init__(self, transactionDetails):
        self._transaction = {}
        self._cache = Cacher({
            'details': {},
        })

        if type(transactionDetails) is dict:
            self._transaction = transactionDetails

            self['timereceived_pretty'] = misc.twitterizeDate(
                self.get('timereceived', 'never'))
            self['time_pretty'] = misc.twitterizeDate(self.get(
                'time', 'never'))
            self['timereceived_human'] = datetime.datetime.fromtimestamp(
                self.get('timereceived', 0))
            self['time_human'] = datetime.datetime.fromtimestamp(
                self.get('time', 0))
            self['blocktime_human'] = datetime.datetime.fromtimestamp(
                self.get('blocktime', 0))
            self['blocktime_pretty'] = misc.twitterizeDate(
                self.get('blocktime', 'never'))
            self['currency_symbol'] = misc.getCurrencySymbol(
                connector, self['currency'])

            if self.get('category', False) in ['receive', 'send']:
                if self['confirmations'] <= MainConfig['globals'][
                        'confirmation_limit']:
                    self['status_icon'] = 'glyphicon-time'
                    self['status_color'] = '#AAA'
                    self['tooltip'] = self['confirmations']
                else:
                    self['status_icon'] = 'glyphicon-ok-circle'
                    self['status_color'] = '#1C9E3F'
                    self['tooltip'] = self['confirmations']

            accountObject = self['wallet'].getAccountByName(self['account'])
            self['account'] = accountObject

            if self['category'] == 'receive':
                self['icon'] = 'glyphicon-circle-arrow-down'
            elif self['category'] == 'send':
                self['icon'] = 'glyphicon-circle-arrow-up'
            elif self['category'] == 'move':
                self['icon'] = 'glyphicon-circle-arrow-right'
                self['otheraccount'] = self['wallet'].getAccountByName(
                    self['otheraccount'])

    def __getitem__(self, key):
        '''
        Getter for dictionary-line behavior
        '''
        if key == "account":
            if self.haskey("account"):
                return self._transaction['account']
            else:
                return self._transaction['details'][0]['account']
        elif key == "category":
            if self.haskey("category"):
                return self._transaction['category']
            else:
                return self._transaction['details'][0]['category']
        elif key == "currency_symbol":
            return self.getCurrencySymbol()
        elif key == "currency_code":
            return self.getCurrencyCode()
        elif key == "raw_transaction":
            return self.getRawTransaction()
        elif key == "source_address":
            return self.getSenderAddress()
        elif key == "address":
            return CoinAddress(self._transaction['address'],
                               self._transaction['account'])

        transaction = getattr(self, '_transaction')
        return transaction.get(key, None)

    def __setitem__(self, key, value):
        '''
        Setter for dictionary-line behavior
        '''
        transaction = getattr(self, '_transaction')
        transaction[key] = value
        return setattr(self, '_transaction', transaction)

    def get(self, key, default=False):
        '''
        get() method for dictionary-line behavior
        '''
        if self._transaction.get(key, False):
            return self._transaction.get(key, False)
        else:
            return default

    def haskey(self, key):
        '''
        Check the existence of key
        '''
        if key in self._transaction.keys():
            return True
        else:
            return False

    @property
    def provider_id(self):
        '''
        Property for the provider id
        '''
        return self.get('provider_id', None)

    @property
    def transaction_id(self):
        '''
        Property for the txid
        '''
        return self.get('txid', None)

    @property
    def txid(self):
        '''
        Property for the txid
        '''
        return self.get('txid', None)

    def getParamHash(self, param=""):
        '''
        This function takes a string and calculates a sha224 hash out of it. 
        It is used to hash the input parameters of functions/method in order to 
        uniquely identify a cached result based  only on the input parameters of 
        the function/method call.
        '''
        cache_hash = hashlib.sha224(param).hexdigest()
        return cache_hash

    def metaProperties(self):
        '''
        Return transaction details, like sender address
        '''
        raw_transaction = self.getRawTransaction()

        if self['category'] == 'receive':
            sender_address = self.decodeScriptSig(raw_transaction,
                                                  self['currency'],
                                                  self['wallet'].getNet())
        else:
            sender_address = None

        return {'sender_address': sender_address}

    def getRawTransaction(self):
        '''
        Get the raw transaction dict
        '''
        cache_hash = self.getParamHash("details")
        cached_object = self._cache.fetch('details', cache_hash)
        if cached_object:
            raw_transaction = cached_object
        else:
            raw_transaction = connector.getRawTransaction(
                self['txid'], self['wallet']['provider_id'])
            self._cache.store('details', cache_hash, raw_transaction)
        return raw_transaction

    def getSenderAddress(self):
        '''
        Getter function for the sender address
        '''
        if self['category'] == 'receive':
            meta_properties = self.metaProperties()
            if meta_properties.get('sender_address', False):
                return CoinAddress(meta_properties['sender_address'],
                                   'This is a sender address!')

        return None

    def decodeScriptSig(self, rawtransaction, currency, net='testnet'):
        '''
        Decode input script signature, courtesy of:
        http://bitcoin.stackexchange.com/questions/7838/why-does-gettransaction-report-me-only-the-receiving-address/8864#8864
        '''
        try:
            script_sig = rawtransaction['vin'][0]['scriptSig']['asm']
        except:
            return "not enough info"

        script = script_sig.split()

        h = hashlib.sha256(script[1].decode("hex")).digest()
        ripe160 = hashlib.new('ripemd160')
        ripe160.update(h)
        d = ripe160.digest()

        prefix = self.prefixes[currency.lower()][net]
        address = (prefix + d)

        # calculate checksum
        checksum = hashlib.sha256(
            hashlib.sha256(address).digest()).digest()[:4]

        # build the raw address
        address += checksum

        # encode the address in base58
        encoded_address = misc.b58encode(address)

        return encoded_address

    def getCurrencySymbol(self):
        '''
        Return the Unicode currency symbol
        '''
        return misc.getCurrencySymbol(connector, self.getCurrencyCode())

    def getCurrencyCode(self):
        '''
        Return the currency code
        '''
        return self.get('currency', "").lower()
def say_smarx_and_reset():
    to_say = Cacher.get_message_by_user(slack_users.hes_zeebz, randomize=True)
    Cacher.reset_messages_by_user(slack_users.hes_zeebz, randomize=True)
    return to_say
def test_smarx(_):
    return len(Cacher.get_messages_by_user(slack_users.hes_zeebz)) >= 25
Exemple #26
0
class Anticheat:

    def __init__(self, args):
        """
        Initializes an Anticheat instance.

        [SimpleNamespace or argparse.Namespace] args:
            A namespace-like object representing how and what to compare. An example may look like
            `Namespace(cache=False, local=False, map_id=None, number=50, threshold=20, user_id=None)`
        """

        self.cacher = Cacher(args.cache)
        self.args = args
        if(args.map_id):
            self.users_info = Loader.users_info(args.map_id, args.number)
        if(args.user_id and args.map_id):
            user_info = Loader.user_info(args.map_id, args.user_id)
            self.replays_check = [OnlineReplay.from_map(self.cacher, args.map_id, args.user_id, user_info[args.user_id][0], user_info[args.user_id][1])]

    def run(self):
        """
        Starts loading and detecting replays based on the args passed through the command line.
        """

        if(self.args.local):
            self._run_local()
        elif(self.args.map_id):
            self._run_map()
        else:
            print("Please set either --local (-l) or --map (-m)! ")
            sys.exit(1)

    def _run_local(self):

        args = self.args
        # get all local user replays (used in every --local case)
        replays1 = [LocalReplay.from_path(osr_path) for osr_path in PATH_REPLAYS_USER]

        threshold = args.threshold
        stddevs = args.stddevs

        if(args.map_id and args.user_id):
            # compare every local replay with just the given user + map replay
            comparer = Comparer(threshold, args.silent, replays1, replays2=self.replays_check, stddevs=stddevs)
            comparer.compare(mode="double")
            return
        if(args.map_id):
            # compare every local replay with every leaderboard entry
            replays2 = OnlineReplay.from_user_info(self.cacher, args.map_id, self.users_info)
            comparer = Comparer(threshold, args.silent, replays1, replays2=replays2, stddevs=stddevs)
            comparer.compare(mode="double")
            return

        if(args.single):
            # checks every replay listed in PATH_REPLAYS_USER against every other replay there
            comparer = Comparer(threshold, stddevs, args.silent, replays1)
            comparer.compare(mode="single")
            return
        else:
            # checks every replay listed in PATH_REPLAYS_USER against every replay listed in PATH_REPLAYS_CHECK
            replays2 = [LocalReplay.from_path(osr_path) for osr_path in PATH_REPLAYS_CHECK]
            comparer = Comparer(threshold, args.silent, replays1, replays2=replays2, stddevs=stddevs)
            comparer.compare(mode="double")
            return

    def _run_map(self):

        args = self.args

        threshold = args.threshold
        stddevs = args.stddevs

        # if doing anything online, revalidate cache
        self.cacher.revalidate(args.map_id, self.users_info)

        if(args.map_id and args.user_id): # passed both -m and -u but not -l
            replays2 = OnlineReplay.from_user_info(self.cacher, args.map_id, self.users_info)
            comparer = Comparer(threshold, args.silent, self.replays_check, replays2=replays2, stddevs=stddevs)
            comparer.compare(mode="double")
            return

        if(args.map_id): # only passed -m
            # get all 50 top replays
            replays = OnlineReplay.from_user_info(self.cacher, args.map_id, self.users_info)
            comparer = Comparer(threshold, args.silent, replays, stddevs=stddevs)
            comparer.compare(mode="single")
            return
class CoinTransaction(object):
    '''
    Class for a transaction
    '''
    
    # source: https://github.com/zamgo/PHPCoinAddress/blob/master/README.md
    prefixes = {
                'btc': {'mainnet': '\x00', 'testnet': '\x6f'},
                'ltc': {'mainnet': '\x30', 'testnet': '\x6f'},
                'ftc': {'mainnet': '\x0E', 'testnet': '\x6f'},
                'ppc': {'mainnet': '\x37', 'testnet': '\x6f'},
                'nmc': {'mainnet': '\x34', 'testnet': '\x6f'},
                'nvc': {'mainnet': '\x08', 'testnet': '\x6f'},
                'doge': {'mainnet': '\x30', 'testnet': '\x6f'},
               }
    
    def __init__(self, transactionDetails):
        self._transaction = {}
        self. _cache = Cacher({
                     'details': {},
                     })
        
        if type(transactionDetails) is dict:
            self._transaction = transactionDetails
            
            self['timereceived_pretty'] = misc.twitterizeDate(self.get('timereceived', 'never'))
            self['time_pretty'] = misc.twitterizeDate(self.get('time', 'never'))
            self['timereceived_human'] = datetime.datetime.fromtimestamp(self.get('timereceived', 0))
            self['time_human'] = datetime.datetime.fromtimestamp(self.get('time', 0))
            self['blocktime_human'] = datetime.datetime.fromtimestamp(self.get('blocktime', 0))
            self['blocktime_pretty'] = misc.twitterizeDate(self.get('blocktime', 'never'))
            self['currency_symbol'] = misc.getCurrencySymbol(connector, self['currency'])
            
            if self.get('category', False) in ['receive', 'send']:
                if self['confirmations'] <= MainConfig['globals']['confirmation_limit']:
                    self['status_icon'] = 'glyphicon-time'
                    self['status_color'] = '#AAA';
                    self['tooltip'] = self['confirmations']
                else:
                    self['status_icon'] = 'glyphicon-ok-circle'
                    self['status_color'] = '#1C9E3F';
                    self['tooltip'] = self['confirmations']
            
            accountObject = self['wallet'].getAccountByName(self['account'])
            self['account'] = accountObject
            
            if self['category'] == 'receive':
                self['icon'] = 'glyphicon-circle-arrow-down'
            elif self['category'] == 'send':
                self['icon'] = 'glyphicon-circle-arrow-up'
            elif self['category'] == 'move':
                self['icon'] = 'glyphicon-circle-arrow-right'
                self['otheraccount'] = self['wallet'].getAccountByName(self['otheraccount'])
            
    def __getitem__(self, key):
        '''
        Getter for dictionary-line behavior
        '''
        if key == "account":
            if self.haskey("account"):
                return self._transaction['account']
            else:
                return self._transaction['details'][0]['account']
        elif key == "category":
            if self.haskey("category"):
                return self._transaction['category']
            else:
                return self._transaction['details'][0]['category']
        elif key == "currency_symbol":
            return self.getCurrencySymbol()
        elif key == "currency_code":
            return self.getCurrencyCode()
        elif key == "raw_transaction":
            return self.getRawTransaction()
        elif key == "source_address":
            return self.getSenderAddress()
        elif key == "address":
            return CoinAddress(self._transaction['address'], self._transaction['account'])
        
        transaction = getattr(self, '_transaction')
        return transaction.get(key, None)
     
    def __setitem__(self, key, value):
        '''
        Setter for dictionary-line behavior
        '''
        transaction = getattr(self, '_transaction')
        transaction[key] = value
        return setattr(self, '_transaction', transaction)
    
    def get(self, key, default=False):
        '''
        get() method for dictionary-line behavior
        '''
        if self._transaction.get(key, False):
            return self._transaction.get(key, False)
        else:
            return default
        
    def haskey(self, key):
        '''
        Check the existence of key
        '''
        if key in self._transaction.keys():
            return True
        else:
            return False
        
    @property
    def provider_id(self):
        '''
        Property for the provider id
        '''
        return self.get('provider_id', None)
    
    @property
    def transaction_id(self):
        '''
        Property for the txid
        '''
        return self.get('txid', None)
    
    @property
    def txid(self):
        '''
        Property for the txid
        '''
        return self.get('txid', None)
    
    def getParamHash(self, param=""):
        '''
        This function takes a string and calculates a sha224 hash out of it. 
        It is used to hash the input parameters of functions/method in order to 
        uniquely identify a cached result based  only on the input parameters of 
        the function/method call.
        '''
        cache_hash = hashlib.sha224(param).hexdigest()
        return cache_hash
    
    def metaProperties(self):
        '''
        Return transaction details, like sender address
        '''
        raw_transaction = self.getRawTransaction()
        
        if self['category'] == 'receive':
            sender_address = self.decodeScriptSig(raw_transaction, self['currency'], self['wallet'].getNet())
        else:
            sender_address = None
            
        return {'sender_address': sender_address}
    
    def getRawTransaction(self):
        '''
        Get the raw transaction dict
        '''
        cache_hash = self.getParamHash("details")
        cached_object = self._cache.fetch('details', cache_hash)
        if cached_object:
            raw_transaction = cached_object
        else:
            raw_transaction = connector.getRawTransaction(self['txid'], self['wallet']['provider_id'])
            self._cache.store('details', cache_hash, raw_transaction)
        return raw_transaction

    def getSenderAddress(self):
        '''
        Getter function for the sender address
        '''
        if self['category'] == 'receive':
            meta_properties = self.metaProperties()
            if meta_properties.get('sender_address', False):
                return CoinAddress(meta_properties['sender_address'], 'This is a sender address!')
            
        return None

    def decodeScriptSig(self, rawtransaction, currency, net='testnet'):
        '''
        Decode input script signature, courtesy of:
        http://bitcoin.stackexchange.com/questions/7838/why-does-gettransaction-report-me-only-the-receiving-address/8864#8864
        '''
        try:
            script_sig = rawtransaction['vin'][0]['scriptSig']['asm']
        except:
            return "not enough info"
        
        script = script_sig.split()
        
        h = hashlib.sha256(script[1].decode("hex")).digest()
        ripe160 = hashlib.new('ripemd160')
        ripe160.update(h)
        d = ripe160.digest()
        
        prefix = self.prefixes[currency.lower()][net]
        address = (prefix + d)
        
        # calculate checksum
        checksum = hashlib.sha256(hashlib.sha256(address).digest()).digest()[:4]
        
        # build the raw address
        address += checksum
        
        # encode the address in base58
        encoded_address = misc.b58encode(address)
        
        return encoded_address

    def getCurrencySymbol(self):
        '''
        Return the Unicode currency symbol
        '''
        return misc.getCurrencySymbol(connector, self.getCurrencyCode())
    
    def getCurrencyCode(self):
        '''
        Return the currency code
        '''
        return self.get('currency', "").lower()
Exemple #28
0
class Keiba(Collector):
    def __init__(self, reporter, waiter, outdir, useragent) -> None:
        super(Keiba, self).__init__()
        self.reporter: Reporter = reporter
        self.waiter = waiter
        self.outdir = outdir
        self.useragent = useragent
        self.cacher = Cacher(self.outdir)
        self.semaphore = Semaphore(2)

    async def get_search_page(
        self,
        n: int,
        options: dict = {
            'pid': str,
            'word': str,
            'track[]': str,
            'start_year': str,
            'start_mon': str,
            'end_year': str,
            'end_mon': str,
            'jyo[]': str,
            'kyori_min': str,
            'kyori_max': str,
            'sort': str,
            'list': str,
        }):
        url = 'https://db.netkeiba.com/'
        psuedo_url = f'{url}?{urllib.parse.urlencode(options)}&page=1'
        filename = urllib.parse.quote(psuedo_url + '.html', safe='')

        cache, _ = self.cacher.get(filename)
        if cache:
            search_result = cache
        else:
            await self.waiter.wait(psuedo_url)
            print('fetching', psuedo_url)
            async with aiohttp.request(
                    'post',
                    url=url,
                    headers={
                        'content-type': 'application/x-www-form-urlencoded',
                        'user-agent': self.useragent
                    },
                    data=urllib.parse.urlencode(options)) as req:
                content = await req.read()
                search_result = content.decode(SITE_ENCODING)
                if str(req.url) == url:
                    self.cacher.set(filename, search_result)
                else:
                    print(f'Warning: redirected to {str(req.url)}')
                    self.cacher.set(
                        urllib.parse.quote(str(req.url), safe='') + '.html',
                        search_result)
                    return None

        if n == 1:
            return search_result
        else:
            data = await self.run_in_executor(get_nextpage_data, search_result)
            data['page'] = str(n)
            psuedo_url = f'{url}?{urllib.parse.urlencode(options)}&page={n}'
            filename = urllib.parse.quote(psuedo_url + '.html', safe='')

            cache, _ = self.cacher.get(filename)
            if cache:
                result = cache
            else:
                await self.waiter.wait(psuedo_url)
                print('fetching', psuedo_url)
                async with aiohttp.request(
                        'post',
                        url=url,
                        headers={
                            'content-type':
                            'application/x-www-form-urlencoded',
                            'user-agent': self.useragent
                        },
                        data=urllib.parse.urlencode(
                            data, encoding=SITE_ENCODING)) as req:
                    try:
                        content = await req.read()
                        result = content.decode(SITE_ENCODING)
                        self.cacher.set(filename, result)
                    except Exception as e:
                        print('get_tail', e)
                        return None
            return result

    async def get_race_page(self, url):
        filename = urllib.parse.quote(url, safe='') + '.html'
        cache, _ = self.cacher.get(filename, ext='')
        if cache:
            html = cache
        else:
            await self.waiter.wait(url)
            print('fetching', url)
            async with aiohttp.request('get',
                                       url,
                                       headers={'user-agent':
                                                self.useragent}) as req:
                content = await req.read()
                html = content.decode(SITE_ENCODING)
                self.cacher.set(filename, html)

        return html

    async def collect(self, year, queue_size=3):
        async def f(page):
            print(page)
            html, _ = await self.async_retry(
                3, self.get_search_page, page, {
                    'pid': 'race_list',
                    'start_year': str(year),
                    'end_year': str(year),
                    'sort': 'date',
                    'list': '100'
                })
            return len([
                await self.add_future('get_race', self.get_race_page(race_url))
                for race_url in await self.run_in_executor(
                    get_race_urls, html)
            ]) == 100 if html else False

        await self.queued_paging(1,
                                 1000,
                                 lambda page: f(page),
                                 queue_size=queue_size)

    async def collect_horse(self, year, queue_size=3):
        async def f(page):
            print(page)
            html, error = await self.async_retry(3, self.get_search_page, page,
                                                 {
                                                     'pid': 'horse_list',
                                                     'list': '100',
                                                     'birthyear': year,
                                                 })
            if error:
                print('Waringn: max retries exceeded')
                return False
            return len(await self.run_in_executor(
                get_horse_urls, html)) == 100 if html else False

        await self.queued_paging(1,
                                 1000,
                                 lambda page: f(page),
                                 queue_size=queue_size)
Exemple #29
0
 def __init__(self, api):
     self.api = api
     self.cache = Cacher('cache2.txt')
Exemple #30
0
class Circleguard:
    def __init__(self, args):
        """
        Initializes a Circleguard instance.

        [SimpleNamespace or argparse.Namespace] args:
            A namespace-like object representing how and what to compare. An example may look like
            `Namespace(cache=False, local=False, map_id=None, number=50, threshold=20, user_id=None)`
        """

        # get all replays in path to check against. Load this per circleguard instance or users moving files around while the gui is open doesn't work.
        self.PATH_REPLAYS = [
            join(PATH_REPLAYS_STUB, f) for f in os.listdir(PATH_REPLAYS_STUB)
            if isfile(join(PATH_REPLAYS_STUB, f)) and f != ".DS_Store"
        ]

        self.cacher = Cacher(args.cache)
        self.args = args
        if (args.map_id):
            self.users_info = Loader.users_info(args.map_id, args.number)
        if (args.user_id and args.map_id):
            user_info = Loader.user_info(
                args.map_id, args.user_id
            )[args.
              user_id]  # should be guaranteed to only be a single mapping of user_id to a list
            self.replays_check = [
                OnlineReplay.from_map(self.cacher, args.map_id, args.user_id,
                                      user_info[0], user_info[1], user_info[2])
            ]

    def run(self):
        """
        Starts loading and detecting replays based on the args passed through the command line.
        """
        if (self.args.verify):
            self._run_verify()
        elif (self.args.local):
            self._run_local()
        elif (self.args.map_id):
            self._run_map()
        else:
            print(
                "Please set either --local (-l), --map (-m), or --verify (-v)! "
            )

    def _run_verify(self):
        args = self.args

        map_id = self.args.verify[0]
        user1_id = self.args.verify[1]
        user2_id = self.args.verify[2]

        user1_info = Loader.user_info(map_id, user1_id)
        user2_info = Loader.user_info(map_id, user2_id)
        replay1 = OnlineReplay.from_user_info(self.cacher, map_id, user1_info)
        replay2 = OnlineReplay.from_user_info(self.cacher, map_id, user2_info)

        comparer = Comparer(args.threshold,
                            args.silent,
                            replay1,
                            replays2=replay2,
                            stddevs=args.stddevs)
        comparer.compare(mode="double")

    def _run_local(self):

        args = self.args
        # get all local replays (used in every --local case)
        replays1 = [
            LocalReplay.from_path(osr_path) for osr_path in self.PATH_REPLAYS
        ]

        threshold = args.threshold
        stddevs = args.stddevs

        if (args.map_id and args.user_id):
            # compare every local replay with just the given user + map replay
            comparer = Comparer(threshold,
                                args.silent,
                                replays1,
                                replays2=self.replays_check,
                                stddevs=stddevs)
            comparer.compare(mode="double")
            return
        if (args.map_id):
            # compare every local replay with every leaderboard entry
            replays2 = OnlineReplay.from_user_info(self.cacher, args.map_id,
                                                   self.users_info)
            comparer = Comparer(threshold,
                                args.silent,
                                replays1,
                                replays2=replays2,
                                stddevs=stddevs)
            comparer.compare(mode="double")
            return
        else:
            comparer = Comparer(threshold,
                                args.silent,
                                replays1,
                                stddevs=stddevs)
            comparer.compare(mode="single")

    def _run_map(self):

        args = self.args

        threshold = args.threshold
        stddevs = args.stddevs

        # if doing anything online, revalidate cache
        self.cacher.revalidate(args.map_id, self.users_info)

        if (args.map_id and args.user_id):  # passed both -m and -u but not -l
            replays2 = OnlineReplay.from_user_info(self.cacher, args.map_id,
                                                   self.users_info)
            comparer = Comparer(threshold,
                                args.silent,
                                self.replays_check,
                                replays2=replays2,
                                stddevs=stddevs)
            comparer.compare(mode="double")
            return

        if (args.map_id):  # only passed -m
            # get all 50 top replays
            replays = OnlineReplay.from_user_info(self.cacher, args.map_id,
                                                  self.users_info)
            comparer = Comparer(threshold,
                                args.silent,
                                replays,
                                stddevs=stddevs)
            comparer.compare(mode="single")
            return
Exemple #31
0
class NaiveBFS:
    def __init__(self, api):
        self.api = api
        self.cache = Cacher('cache2.txt')

    def bfs(self, start_user, target_user):
        """
        Find shortest path from start_user to target_user, where a path is defined as:
        X --> Y when user X follows user Y.
        This is a horrible algorithm - don't use it unless you're willing to wait for around a week or longer
        if you know the degree of separation is >= 2
        :param start_user: The starting (origin) user
        :param target_user: The user you're trying to find a path to
        :return: A path from start_user to target_user
        """
        api = self.api
        error_handler = ErrorHandler(api=api)
        queue = Queue()
        queue.put(start_user)
        visited = {start_user}
        path_to = defaultdict(lambda: [start_user])

        while queue.not_empty:
            current_user = queue.get()
            if self.cache.read_from_cache(
                    user_id=current_user
            ) is None:  # current_user does not yet exist in cache
                current_user_followings = api.GetFriendIDs(
                    user_id=current_user)
                current_user_screen_name = api.GetUser(
                    user_id=current_user).name
                self.cache.append_to_cache(
                    user_id=current_user,
                    screen_name=current_user_screen_name,
                    following_ids=current_user_followings)
                print(
                    f"Naive BFS: requested {current_user_screen_name}'s screen name and followings from API. "
                    f"Commencing search.")
            else:
                cached_data = self.cache.read_from_cache(user_id=current_user)
                current_user_screen_name = cached_data[
                    'screen_name']  # if user_id exists as a key in cache,
                # screen_name will always exist as well
                if not cached_data[
                        'following']:  # i.e. following list does not yet exist in cache
                    current_user_followings = api.GetFriendIDs(
                        user_id=current_user)
                    self.cache.append_to_cache(
                        user_id=current_user,
                        screen_name=current_user_screen_name,
                        following_ids=current_user_followings)
                    print(
                        f"Naive BFS: retrieved {current_user_screen_name}'s screen name from cache. "
                        f"Requested followings from API. Commencing search.")
                else:
                    current_user_followings = cached_data['following']
                    print(
                        f"Naive BFS: retrieved {current_user_screen_name}'s screen name AND followings from cache. "
                        f"Today is a good day. Commencing search.")

            num_followings = len(current_user_followings)
            for num, following in enumerate(current_user_followings):
                if following not in visited:
                    try:
                        if self.cache.read_from_cache(
                                user_id=following
                        ) is None:  # this user does not yet exist in cache
                            following_screen_name = api.GetUser(
                                user_id=following).name
                            self.cache.append_to_cache(
                                user_id=following,
                                screen_name=following_screen_name)
                            print(
                                f"Naive BFS: {current_user_screen_name} follows {following_screen_name}. "
                                f"Requested {following_screen_name}'s screen name from API and cached."
                            )
                        else:
                            following_screen_name = self.cache.read_from_cache(
                                user_id=following)['screen_name']
                            print(
                                f"Naive BFS: {current_user_screen_name} follows {following_screen_name}. "
                                f"Retrieved {following_screen_name}'s screen name from cache."
                            )

                        visited.add(following)
                        queue.put(following)
                        current_path = list(path_to[current_user])
                        current_path.extend([following])
                        path_to[following] = current_path
                        if following == target_user:
                            print(
                                f"Naive BFS: {following_screen_name} (target user) found! Stopping search."
                            )
                            break
                    except twitter.error.TwitterError as ex:
                        error_handler.handle(ex, user_id=following)
                else:
                    print('User already visited. Skipping...')
                print(num + 1, '/', num_followings)
            else:
                continue
            break
        if path_to[target_user] != [start_user] or target_user == start_user:
            return path_to[target_user]
        else:
            return None

    def display_paths(self, path):
        """
        Prints out path in English
        :param path: List of users in path
        """
        for idx, user in enumerate(path):
            screen_name = self.cache.read_from_cache(
                user_id=user)['screen_name']
            if idx == 0:
                print(f'{screen_name} follows ', end='')
            elif idx == len(path) - 1:
                print(screen_name)
            else:
                print(f'{screen_name}, who follows ', end='')

    def run_search(self, origin_user, target_user):
        """
        Driver code to run search
        :param origin_user: User to start search from
        :param target_user: User to find path to
        :return: List of users in possible path from origin_user to target_user
        """
        found_path = self.bfs(start_user=origin_user, target_user=target_user)
        self.display_paths(path=found_path)
        return found_path
Exemple #32
0
class CoinWallet(object):
    '''
    Class for a wallet
    '''
    def __init__(self, wallet_config):
        self._errors = []
        self._config = {}
        self._cache = Cacher({
             'accounts': {},
             'transactions': {},
             'transactiondetails': {},
             'balances': {},
             'info': {},
             })
        
        if type(wallet_config) is dict:
            self._config = wallet_config
        
    @property
    def provider_id(self):
        '''
        Property for the provider id
        '''
        return self.get('id', None)
    
    @property
    def enabled(self):
        '''
        Return the status (enabled/disabled)
        '''
        return self.get('enabled', False)
    
    def __getitem__(self, key):
        '''
        Getter for dictionary-line behavior
        '''
        if key == "currency_symbol":
            return self.getCurrencySymbol()
        elif key == "currency_code":
            return self.getCurrencyCode()
        
        config = getattr(self, '_config')
        return config[key]
     
    def __setitem__(self, key, value):
        '''
        Setter for dictionary-line behavior
        '''
        config = getattr(self, '_config')
        config[key] = value
        return setattr(self, '_config', config)
    
    def get(self, key, default=False):
        '''
        Implementing .get() method for dictionary-line behavior
        '''
        if self._config.get(key, False):
            return self._config.get(key, False)
        else:
            return default
    
    def haskey(self, key):
        '''
        Check the existence of key
        '''
        if key in self._config.keys():
            return True
        else:
            return False
    
    def getNet(self):
        '''
        Return network value, mainnet or testnet
        '''
        # check for cached data, use that or get it again
        cache_hash = self.getParamHash("provider_id=%s" % self.provider_id)
        cached_peerinfo = self._cache.fetch('info', cache_hash)
        if cached_peerinfo:
            info = cached_peerinfo
        else:
            info = connector.getInfo(self.provider_id)
            self._cache.store('info', cache_hash, info)
        
        is_testnet = False
        if info.has_key('testnet'):
            is_testnet = info.get('testnet')
            if is_testnet is False:
                return "mainnet"
            elif is_testnet is True:
                return "testnet"
        else:
            # default to mainnet
            return "mainnet"
    
    def balance(self):
        '''
        Get wallet balance
        '''
        # check for cached data, use that or get it again
        cache_hash = self.getParamHash("balance")
        cached_object = self._cache.fetch('balance', cache_hash)
        if cached_object:
            return cached_object.get(self.provider_id, "-")
        
        balance = connector.getBalance(self.provider_id)

        # store result in cache
        self._cache.store('balance', cache_hash, balance)
        
        return misc.longNumber(balance.get(self.provider_id, "-"))
    
    def getParamHash(self, param=""):
        '''
        This function takes a string and calculates a sha224 hash out of it. 
        It is used to hash the input parameters of functions/method in order to 
        uniquely identify a cached result based  only on the input parameters of 
        the function/method call.
        '''
        cache_hash = hashlib.sha224(param).hexdigest()
        return cache_hash
    
    def listAccounts(self, gethidden=False, getarchived=False):
        '''
        Get a list of accounts. This method also supports filtering, fetches address for each account etc.
        '''
        # check for cached data, use that or get it again
        cache_hash = self.getParamHash("gethidden=%s&getarchived=%s" % (gethidden, getarchived))
        cached_object = self._cache.fetch('accounts', cache_hash)
        if cached_object:
            return cached_object
        
        # get data from the connector (xxxcoind)
        fresh_accounts = connector.listAccounts(gethidden=False, getarchived=False, selected_provider_id=self.provider_id)

        # get a list of archived address
        address_ignore_list = []
        if not getarchived:
            ignore_list = accountFilter.objects.filter(status=1)
            for ignored_account in ignore_list:
                address_ignore_list.append(ignored_account.address.encode('ascii'))
        
        # get a list of hidden accounts
        address_hidden_list = []
        if not gethidden:
            hidden_list = accountFilter.objects.filter(status=2)
            for hidden_account in hidden_list:
                address_hidden_list.append(hidden_account.address.encode('ascii'))
        
        accountObjects = []
        for account_name, account_balance in fresh_accounts.get(self.provider_id, {}).items():

            '''
            # check all addresses if they are in the archive list
            for ignored_address in address_ignore_list:
                if ignored_address in account_addresses:
                    del account_addresses[account_addresses.index(ignored_address)]
            
            # check all addresses if they are in the hidden list
            hidden_flag = False
            for hidden_address in address_hidden_list:
                if hidden_address in account_addresses:
                    hidden_flag = True
            '''
            
            accountObjects.append(CoinAccount({
                                       'name': account_name,
                                       'balance': account_balance,
                                       'currency': self['currency'],
                                       'provider_id': self.provider_id,
                                       'wallet': self,
                                       }))
        
        # cache the result
        self._cache.store('accounts', cache_hash, accountObjects)
        return accountObjects
    
    def getCurrencySymbol(self):
        '''
        Return the Unicode currency symbol
        '''
        return self.get('symbol', None)
    
    def getCurrencyCode(self):
        '''
        Return the currency code
        '''
        return self.get('currency', "").lower()
    
    def listTransactions(self, limit=10, start=0):
        '''
        Return a list of transactions wallet-wide
        '''
        # check for cached data, use that or get it again
        cache_hash = self.getParamHash("limit=%s&start=%s" % (limit, start))
        cached_object = self._cache.fetch('transactions', cache_hash)
        if cached_object:
            return cached_object
        
        transactions = []
        transactions_dicts = connector.listTransactionsByAccount("*", self.provider_id, limit, start)
        for transaction in transactions_dicts:
            transaction['wallet'] = self
            transaction['currency'] = self.getCurrencyCode()
            transaction['currency_symbol'] = self.getCurrencySymbol()
            transaction['provider_id'] = self.provider_id
            transactions.append(CoinTransaction(transaction))
        
        self._cache.store('transactions', cache_hash, transactions)
        return transactions
    
    def getAccountByName(self, name):
        '''
        Return CoinAccount() for name
        '''
        accounts = self.listAccounts(gethidden=True, getarchived=True)
        for account in accounts:
            if account['name'] == name:
                return account
        
        return None
    
    def getTransactionById(self, txid):
        '''
        Return a transaction by txid
        '''
        # check for cached data, use that or get it again
        cache_hash = self.getParamHash("txid=%s" % (txid))
        cached_object = self._cache.fetch('transactiondetails', cache_hash)
        if cached_object:
            return cached_object
        
        transaction_details = connector.getTransaction(txid, self.provider_id)
        transaction_details['currency'] = self.getCurrencyCode()
        transaction_details['wallet'] = self
        
        self._cache.store('transactiondetails', cache_hash, transaction_details)
        return CoinTransaction(transaction_details)
    
    def getAddressesByAccount(self, account):
        '''
        Get a list of address for account
        '''
        addresses_list = connector.getAddressesByAccount(account, self.provider_id)
        coinaddresses = []
        for address in addresses_list:
            coinaddresses.append(CoinAddress(address, account))
            
        return coinaddresses
    
    def getDefaultAccount(self):
        '''
        Return the CoinAccount object for the default wallet account
        '''
        accounts = self.listAccounts(gethidden=True, getarchived=True)
        for account in accounts:
            if len(account['name']) == 0:
                return account
        else:
            return None
        
    def getAccountByAddress(self, address):
        '''
        Return account by address
        '''
        accounts = self.listAccounts(gethidden=True, getarchived=True)
        
        target_account = None
        for account in accounts:
            for account_address in account['addresses']:
                if str(address) == str(account_address):
                    target_account = account
                    target_account['currency'] = self.getCurrencyCode()
                    target_account['provider_id'] = self.provider_id
                    return target_account
        else:
            return None
        
    def getAccountByIdentifier(self, identifier):
        '''
        Get account by identifier
        '''
        list_of_accounts = self.listAccounts(gethidden=True, getarchived=True)
        for account in list_of_accounts:
            if account.getIdentifier() == identifier:
                return account
        else:
            return None
Exemple #33
0
 else:
     if event_type == "resourceDown":
         if reason is None:
             print "Recieved resource down handle, but missing reason."
             print __doc__
             sys.exit(2)
         if share_down is None:
             print "Recieved resource down handle, but missing share down."
             print __doc__
             sys.exit(2)
         if date_down is None:
             print "Recieved resource down handle, but missing date down."
 # We have everything we require to create an event.
 # Attempt to find already cached data:
 m = MetaDoc(site_name)
 c = Cacher("events")
 cached_data = c.get_cache()
 if cached_data is not None:
     processor = Events.from_xml_element(cached_data, Events)
     if processor is None:
         print "Found previous event cache, but could not load. Please check "
         print "\"%s\" for errors. " % c.file_path
         print "Halting."
         sys.exit(2)
     else:
         c.remove_cache()
 else:
     processor = Events()
 if event_type == "resourceUp":
     e = ResourceUpEntry(date_up, reason, remarks)
 else:
Exemple #34
0
 def __init__(self, getterType='urlopen', waitTimeBeforeScraping=0):
     self.cacher = Cacher()
     self.config = Config()
     self.getter = Getter(getterType,
                          waitTimeBeforeScraping=waitTimeBeforeScraping)
class CoinAccount(object):
    '''
    Class for an account
    '''
    
    def __init__(self, accountDetails):
        self._errors = []
        self._account = {}
        self._hidden = False
        self._cache = Cacher({
         'transactions': {},
         'balances': {},
         'addressesbyaccount': {},
         })
        
        if type(accountDetails) is dict:
            self._account = accountDetails
            self._provider_id = accountDetails['provider_id']
            
    @property
    def provider_id(self):
        '''
        Property for the provider id
        '''
        return self.get('provider_id', None)
           
    def __getitem__(self, key):
        '''
        Getter for dictionary-line behavior
        '''

        if key == "addresses":
            return self.getAddresses()
        elif key == "last_activity":
            return self.getLastActivity()
        elif key == "currency_symbol":
            return self.getCurrencySymbol()
        elif key == "currency_code":
            return self.getCurrencyCode()
        elif key == 'identifier':
            return self.getIdentifier()

        account = getattr(self, '_account')
        return account.get(key, None)
     
    def __setitem__(self, key, value):
        '''
        Setter for dictionary-line behavior
        '''
        account = getattr(self, '_account')
        account[key] = value
        return setattr(self, '_account', account)
    
    def get(self, key, default=False):
        '''
        Getter for dictionary-line behavior
        '''
        if self._account.get(key, False):
            return self._account.get(key, False)
        else:
            return default
        
    def haskey(self, key):
        '''
        Check the existence of key
        '''
        if key in self._account.keys():
            return True
        else:
            return False
    
    def getParamHash(self, param=""):
        '''
        This function takes a string and calculates a sha224 hash out of it. 
        It is used to hash the input parameters of functions/method in order to 
        uniquely identify a cached result based  only on the input parameters of 
        the function/method call.
        '''
        cache_hash = hashlib.sha224(param).hexdigest()
        return cache_hash
    
    def getIdentifier(self):
        '''
        There is no unique identifier for an account in a xxxcoind daemon
        so lets make one. Hopefully the below hashing method will uniquely 
        identify an account for us
        '''
        unique_string = "provider_id=%s&name=%s&currency=%s" % (self['provider_id'], self['name'], self['currency'])
        identifier = hashlib.sha1(unique_string).hexdigest()
        return identifier
        
    def isDefault(self):
        '''
        Return bool whether this is a default account or not
        '''
        if self._account['name'] == u"":
            self._hidden = True
            return True
        else:
            return False
    
    def getBalance(self):
        '''
        Return the account balance
        '''
        balance = connector.getBalance(self.provider_id, self['name'])
        return misc.longNumber(balance)
    
    def isHidden(self):
        '''
        Return bool if this account is hidden
        '''
        return self._hidden or self._account['hidden'] or self.isDefault()
    
    def getAddresses(self):
        '''
        Get the address for an account name
        '''
        # check for cached data, use that or get it again
        cache_hash = self.getParamHash("name=%s" % (self['name']))
        cached_object = self._cache.fetch('addressesbyaccount', cache_hash)
        if cached_object:
            return cached_object
        
        addresses = connector.getAddressesByAccount(self['name'], self.provider_id)
        addresses_list = []
        for address in addresses:
            coinaddr = CoinAddress(address, self)
            addresses_list.append(coinaddr)
            
        # cache the result
        self._cache.store('addressesbyaccount', cache_hash, addresses_list)
        return addresses_list
    
    def getAddressesCount(self):
        '''
        Return the number of address under this account
        '''
        addresses = self.getAddresses()
        return len(addresses)
    
    def getLastActivity(self):
        '''
        Return the date of the last activity
        '''
        last_transaction = self.listTransactions(1, 0)
        if last_transaction:
            last_activity = misc.twitterizeDate(last_transaction[0]['time'])
        else:
            last_activity = "never"
            
        self['last_activity'] = last_activity
        return last_activity
    
    def getCurrencySymbol(self):
        '''
        Return the Unicode currency symbol
        '''
        return misc.getCurrencySymbol(connector, self.getCurrencyCode())
    
    def getCurrencyCode(self):
        '''
        Return the currency code
        '''
        return self.get('currency', "").lower()
    
    def listTransactions(self, limit=100000, start=0, orderby='time', reverse=True):    
        '''
        Get a list of transactions by account name and provider_id
        '''

        cache_hash = self.getParamHash("limit=%s&start=%sorderby=%s&reverse=%s" % (limit, start, orderby, reverse))
        cached_object = self._cache.fetch('transactions', cache_hash)
        if cached_object:
            return cached_object
        
        transactions = []
        transaction_list = connector.listTransactionsByAccount(self['name'], self['provider_id'], limit, start)
        
        for entry in transaction_list:
            if entry.get('address', False):
                entry['address'] = CoinAddress(entry['address'], self)
            
            # give out a provider id and a currency code to the transaction dict
            entry['provider_id'] = self.provider_id
            entry['currency'] = self['currency']
            
            if entry['category'] == 'receive':
                entry['source_address'] = CoinAddress(entry.get('details', {}).get('sender_address', False), "This is a sender address!")
            elif entry['category'] == 'send':
                entry['source_addresses'] = self['wallet'].getAddressesByAccount(entry['account'])
            
            entry['wallet'] = self['wallet']
            
            coin_transaction = CoinTransaction(entry)
            transactions.append(coin_transaction)
            
        # sort result
        transactions = sorted(transactions, key=lambda transaction: transaction[orderby], reverse=reverse) 
            
        # cache the result
        self._cache.store('transactions', cache_hash, transactions)
        return transactions
def process_message(data, slack_client):
    global bot_message_count
    print "process message", data
    data = Data(**data)

    global slack_users
    slack_users = SlackUsers.generate(slack_client)
    try:
        if slack_users.hes_zeebz == data.user:
            Cacher.add_message_by_user(slack_users.hes_zeebz, data.text)
    except Exception:
        pass

    get_or_set_bot_messages(data, slack_client)
    print "BOT MESSAGE COUNT", bot_message_count

    # We don't care what Phteven Marquarst says
    if data.user == slack_users.phteven:
        print "its Phteven..."
        return

    rules = PhtevenRules(data)

    rules.add_many_exclusive([

        # If @Phteven gets mentioned
        (
            lambda data: slack_users.phteven.at in data.text,
            lambda: "^ %s" % slack_users.not_stephanie.at
        ),
        # Copy smarx
        (
            lambda data: random.random() < 0.05 and data.user == slack_users.hes_zeebz,
            lambda: data.text
        ),
        # Smarx has an elipsis
        (
            lambda data: u"\u2026" in data.text and data.user == slack_users.hes_zeebz,
            lambda: "Nice try, Steven J Smarx"
        ),
        # Are we in burgertory?
        (
            lambda data: 'burgertory' in data.text.lower() or 'burgatory' in data.text.lower(),
            lambda: "We're always in burgertory"
        ),
        # Zeebs trying to be funny
        (
            lambda data: PhtevenRegex.thats_something.search(data.text) and data.user == slack_users.whos_zeebz,
            lambda: "Shut up, %s" % slack_users.whos_zeebz.at
        ),
        # Am i saying meow?
        (
            lambda data: PhtevenRegex.meow_now.search(data.text),
            lambda: "Right meow?"
        ),
        # That's <blank>
        (
            lambda data: PhtevenRegex.thats_something.search(data.text),
            lambda: "You're %s" % PhtevenRegex.thats_something.search(data.text).group(2)
        ),
        # You're <blank>
        (
            lambda data: PhtevenRegex.youre_something.search(data.text),
            lambda: "No, you're %s!!" % PhtevenRegex.youre_something.search(data.text).group(2)
        ),
        # Childish. References to poop things
        (
            lambda data: any(word in ['poop', 'turd'] for word in data.text.split(' ')),
            lambda: ":poop:"
        ),
        # if someone talks about Seattle
        (
            lambda data: "seattle" in data.text.lower(),
            lambda: "Seattle?? Isn't that where %s lives?" % slack_users.whos_zeebz.at
        ),
        # Vegas??
        (
            lambda data: "vegas" in data.text.lower(),
            lambda: "%s lives in Vegas!" % slack_users.hes_zeebz.at
        ),
        # If someone just says hello
        (
            lambda data: 'hello' in data.text.lower() and data.user == 'U12D20DCP',
            lambda: "Traduccion: Hola."
        ),
        # Did someone say snackatizer??
        (
            lambda data: "snack" in data.text,
            lambda: "Mayhaps mean snackatizer???"
        ),
        # Myth busted
        (
            lambda data: PhtevenRegex.myth_busted.search(data.text),
            lambda: random.choice(PhtevenPhrases.myth_busted_phrases)
        ),
        # What dis is?
        (
            lambda data: PhtevenRegex.whatdisis.search(data.text),
            lambda: "YOU KNOW WHAT DIS IS"
        ),
        # If something isn't real
        (
            lambda data: PhtevenRegex.arentreal.search(data.text),
            lambda: "You're not real."
        ),
    ])

    rules.add_many_inclusive([
        (
            check_bot_message_count,
            lambda: "TRIPLE BOT MESSAGE COMBO!!!"
        ),
        # User id?
        (
            who_is,
            lambda: sendback_users(data)
        ),
        # Science questions
        (
            lambda data: random.random() < 0.03,
            say_something_sciencey
        ),
        # room, oasted
        (
            room_oasted,
            lambda: say_room_oasted(data)
        ),
        # That's what she said
        (
            lambda data: "harder" in data.text,
            lambda: "That's what she said"
        ),
        # Say what smarx said...but a while ago
        (
            test_smarx,
            say_smarx_and_reset
        ),
        # Say what stephen said...but a while ago
        (
            test_stephen,
            say_stephen_and_reset
        ),
        # Party pooper
        (
            lambda data: 'party' in data.text.lower(),
            lambda: "More like party pooper. lawl"
        ),
        # Smarx was president of the juggling club
        (
            lambda data: 'juggl' in data.text.lower(),
            lambda: "Did you know %s was president of his juggling club in school?" % slack_users.hes_zeebz.at
        ),
        # WAHH
        (
            lambda data: random.random() < 0.02,
            lambda: random.choice(PhtevenPhrases.phteven_phrases)
        ),
        # As it were
        (
            lambda data: 'as it were' in data.text.lower(),
            lambda: "Pindeed"
        ),
        # I hardly knew her
        (
            lambda data: PhtevenRegex.baylor.search(data.text),
            lambda: "%s, I hardly know her" % PhtevenRegex.baylor.search(data.text).group(1)
        ),
        # Be quiet, zeebz
        (
            lambda data: (data.user == slack_users.whos_zeebz) and random.random() < 0.01,
            lambda: "Shut up, %s" % slack_users.whos_zeebz.at
        ),
        # Foldable bluetooth keyboard
        (
            lambda data: 'foldable bluetooth keyboard' in data.text.lower(),
            lambda: "You're a foldable bluetooth keyboard"
        ),
        # Giphy
        (
            lambda data: data.text.startswith('/giphy') and random.random() < 0.50,
            lambda: random.choice(PhtevenPhrases.giphy_phrases)
        ),
        # Hello
        (
            lambda data: 'hello' in data.text.lower(),
            lambda: "Oh. Hello."
        ),
    ])

    processor = PhtevenProcessor(rules)
    outputs.extend(processor.process())
Exemple #37
0
    print("文章来源:" + D["SOURCE"])
    print("采集时间:" + D["PickingTime"])
    print("正文内容:" + D["CONTENT"].decode("utf-8"))
    return D


def showMenu():
    print(
        "[1]文章点赞(Vote)\t[2]文章分享(Share)\t[3]文章保存(Save)\n\
		[4]文章评论(Remark)\t[5]邮件反馈(Email)\t[ENTER]下一篇"
    )
    cmd = input("请输入指令(Q退出):")
    return cmd


cc = Cacher()
cc.start()
input("\t欢迎使用简致阅读\n\t按任意件开启阅读")

D = showArticle()
cmd = showMenu()
while True:
    if cmd == "Q":
        break
    elif cmd == "1":
        pass
    elif cmd == "2":
        pass
    elif cmd == "3":
        pass
    elif cmd == "4":
def test_stephen(_):
    return len(Cacher.get_messages_by_user(slack_users.not_stephanie)) >= 25
def say_stephen_and_reset():
    to_say = Cacher.get_message_by_user(slack_users.not_stephanie, randomize=True)
    Cacher.reset_messages_by_user(slack_users.not_stephanie, randomize=True)
    return to_say
Exemple #40
0
class Extractor(object):
    def __init__(self):
        self._cache_enabled = False
        self._cache_connection_port = None
        self._cache_connection_host = None
        self._cacher = None
        self._arg = {
            "country_codes": ["hrv", "usa"],
            "indicator_codes": ["SP.POP.TOTL"],
            "start_date": 1980,
            "end_date": 2010,
            "pause": 0
        }

    def arg(self):
        return copy.deepcopy(self._arg)

    def normalize(self, arg):
        if arg == None:
            arg = self.arg()
        if "interval" in arg:
            arg["start_date"], arg["end_date"] = arg["interval"]
        capitalize_list = lambda elems: [el.upper() for el in elems]
        arg["country_codes"] = capitalize_list(arg["country_codes"])
        return arg

    def grab(self, arg=None):
        rca_indicators = []
        wb_indicators = []

        try:
            import local_storage.rca.api
            # Check if indicator is in RCA dataset, if not then it must be in WB.
            # TODO: Make indicator type checking universal.
            available_rca_indicators = set(
                local_storage.rca.api.all_indicators())
            for indicator_code in arg["indicator_codes"]:
                if indicator_code in available_rca_indicators:
                    rca_indicators.append(indicator_code)
                else:
                    wb_indicators.append(indicator_code)

            # Grab RCA data.
            rca_arg = copy.deepcopy(arg)
            rca_arg["indicator_codes"] = rca_indicators
            rca_countries = self._grab_from_api(rca_arg, local_storage.rca.api)
        except ImportError:
            print(
                "Can't import local_storage.rca.api! Will assume there aren't any indicators from there"
            )
            rca_countries = []
            wb_indicators = arg["indicator_codes"]

        # Grab WB data.
        wb_arg = copy.deepcopy(arg)
        wb_arg["indicator_codes"] = wb_indicators
        wb_countries = self._grab_from_api(wb_arg, wb.api)

        #TODO: Separate function for merging two or more list of countries.
        # Not assuming that wb and rca countries are in corresponding order in lists.
        all_codes = set()
        wb_dict = {}
        for wb_country in wb_countries:
            all_codes.add(wb_country.code)
            wb_dict[wb_country.code] = wb_country
        rca_dict = {}
        for rca_country in rca_countries:
            all_codes.add(rca_country.code)
            rca_dict[rca_country.code] = rca_country
        combined_countries = []
        for code in all_codes:
            current_countries = []
            try:
                current_countries.append(wb_dict[code])
            except KeyError:
                pass
            try:
                current_countries.append(rca_dict[code])
            except KeyError:
                pass
            if len(current_countries) != 0:
                if len(current_countries) > 1:
                    current_countries[0].merge_with_country(
                        current_countries[1])
                combined_countries.append(current_countries[0])
        return combined_countries

#        #TODO: Merge RCA and WB data.
#        # Assumptions is that rca_countries and wb_countries have elements in same order (maybe write unit test for that).
#        for wb_country,rca_country in zip(wb_countries,rca_countries):
#            wb_country.merge_with_country(rca_country)
#
#        return wb_countries

    def _grab_from_api(self, arg=None, api=wb.api):
        #TODO: think about using kwargs
        """
        possible args are:
        
        1) the default by leaving None, defined in arg()
        2) setting your own such as:
                {"country_codes" : ["hrv", "usa", "chn"],
                   "indicator_codes" : ["SP.POP.TOTL"],
                   "start_date":1980,
                   "end_date":2010}
        3) using interval to set the dates more quickly (overrides start/end_date)
                {"country_codes" : ["hrv", "usa", "chn"],
                   "indicator_codes" : ["SP.POP.TOTL"],
                   "interval":(1980,2010)}
        """
        arg = self.normalize(arg)
        if self._cache_enabled:
            countries = self._cacher.retreive(arg)
            if countries: self.cache_hit = True
            else: self.cache_hit = False
        if not self._cache_enabled or not self.cache_hit:
            countries = api.query_multiple_data(arg["country_codes"],
                                                arg["indicator_codes"],
                                                arg["start_date"],
                                                arg["end_date"])
            if arg["pause"]:
                time.sleep(arg["pause"])
            if self._cache_enabled and not self.cache_hit:
                self._cache(countries)
        return countries

    def grab_metadata(self, what="countries", api=wb.api):
        """
        @param what: string of what to get
         - 'countries' - list of countries
         - 'indicators' - list of indicators
        """
        if what == "countries":
            return api.all_countries()
        elif what == "indicators":
            return api.all_indicators()

    def enable_cache(self, host="localhost", port=27017, test=False):
        """
        @param host: the hostname of the server where mongodb is running
        @param port: port mongodb is listening on
        @param test: True if you want to work on a db separate than the main cache
        """
        self._cache_connection_host = host
        self._cache_connection_port = port
        self._cache_enabled = True
        self._cacher = Cacher(self._cache_connection_host,
                              self._cache_connection_port, test)

    def disable_cache(self):
        self._cache_enabled = False

    def clear_cache(self):
        """
        clear everything from the cache
        """
        self._cacher.clear()

    def _cache(self, countries):
        if self._cache_enabled:
            self._cacher.cache(countries)
        else:
            return

    def is_cached(self, arg):
        arg = self.normalize(arg)
        countries = self._cacher.retreive(arg)
        if countries: return True
        else: return False

    def was_cached(self):
        """
        tells if the data was found in the cache on the last grab
        """
        return self.cache_hit
Exemple #41
0
class CoinAccount(object):
    '''
    Class for an account
    '''
    def __init__(self, accountDetails):
        self._errors = []
        self._account = {}
        self._hidden = False
        self._cache = Cacher({
            'transactions': {},
            'balances': {},
            'addressesbyaccount': {},
        })

        if type(accountDetails) is dict:
            self._account = accountDetails
            self._provider_id = accountDetails['provider_id']

    @property
    def provider_id(self):
        '''
        Property for the provider id
        '''
        return self.get('provider_id', None)

    def __getitem__(self, key):
        '''
        Getter for dictionary-line behavior
        '''

        if key == "addresses":
            return self.getAddresses()
        elif key == "last_activity":
            return self.getLastActivity()
        elif key == "currency_symbol":
            return self.getCurrencySymbol()
        elif key == "currency_code":
            return self.getCurrencyCode()
        elif key == 'identifier':
            return self.getIdentifier()

        account = getattr(self, '_account')
        return account.get(key, None)

    def __setitem__(self, key, value):
        '''
        Setter for dictionary-line behavior
        '''
        account = getattr(self, '_account')
        account[key] = value
        return setattr(self, '_account', account)

    def get(self, key, default=False):
        '''
        Getter for dictionary-line behavior
        '''
        if self._account.get(key, False):
            return self._account.get(key, False)
        else:
            return default

    def haskey(self, key):
        '''
        Check the existence of key
        '''
        if key in self._account.keys():
            return True
        else:
            return False

    def getParamHash(self, param=""):
        '''
        This function takes a string and calculates a sha224 hash out of it. 
        It is used to hash the input parameters of functions/method in order to 
        uniquely identify a cached result based  only on the input parameters of 
        the function/method call.
        '''
        cache_hash = hashlib.sha224(param).hexdigest()
        return cache_hash

    def getIdentifier(self):
        '''
        There is no unique identifier for an account in a xxxcoind daemon
        so lets make one. Hopefully the below hashing method will uniquely 
        identify an account for us
        '''
        unique_string = "provider_id=%s&name=%s&currency=%s" % (
            self['provider_id'], self['name'], self['currency'])
        identifier = hashlib.sha1(unique_string).hexdigest()
        return identifier

    def isDefault(self):
        '''
        Return bool whether this is a default account or not
        '''
        if self._account['name'] == u"":
            self._hidden = True
            return True
        else:
            return False

    def getBalance(self):
        '''
        Return the account balance
        '''
        balance = connector.getBalance(self.provider_id, self['name'])
        return misc.longNumber(balance)

    def isHidden(self):
        '''
        Return bool if this account is hidden
        '''
        return self._hidden or self._account['hidden'] or self.isDefault()

    def getAddresses(self):
        '''
        Get the address for an account name
        '''
        # check for cached data, use that or get it again
        cache_hash = self.getParamHash("name=%s" % (self['name']))
        cached_object = self._cache.fetch('addressesbyaccount', cache_hash)
        if cached_object:
            return cached_object

        addresses = connector.getAddressesByAccount(self['name'],
                                                    self.provider_id)
        addresses_list = []
        for address in addresses:
            coinaddr = CoinAddress(address, self)
            addresses_list.append(coinaddr)

        # cache the result
        self._cache.store('addressesbyaccount', cache_hash, addresses_list)
        return addresses_list

    def getAddressesCount(self):
        '''
        Return the number of address under this account
        '''
        addresses = self.getAddresses()
        return len(addresses)

    def getLastActivity(self):
        '''
        Return the date of the last activity
        '''
        last_transaction = self.listTransactions(1, 0)
        if last_transaction:
            last_activity = misc.twitterizeDate(last_transaction[0]['time'])
        else:
            last_activity = "never"

        self['last_activity'] = last_activity
        return last_activity

    def getCurrencySymbol(self):
        '''
        Return the Unicode currency symbol
        '''
        return misc.getCurrencySymbol(connector, self.getCurrencyCode())

    def getCurrencyCode(self):
        '''
        Return the currency code
        '''
        return self.get('currency', "").lower()

    def listTransactions(self,
                         limit=100000,
                         start=0,
                         orderby='time',
                         reverse=True):
        '''
        Get a list of transactions by account name and provider_id
        '''

        cache_hash = self.getParamHash(
            "limit=%s&start=%sorderby=%s&reverse=%s" %
            (limit, start, orderby, reverse))
        cached_object = self._cache.fetch('transactions', cache_hash)
        if cached_object:
            return cached_object

        transactions = []
        transaction_list = connector.listTransactionsByAccount(
            self['name'], self['provider_id'], limit, start)

        for entry in transaction_list:
            if entry.get('address', False):
                entry['address'] = CoinAddress(entry['address'], self)

            # give out a provider id and a currency code to the transaction dict
            entry['provider_id'] = self.provider_id
            entry['currency'] = self['currency']

            if entry['category'] == 'receive':
                entry['source_address'] = CoinAddress(
                    entry.get('details', {}).get('sender_address', False),
                    "This is a sender address!")
            elif entry['category'] == 'send':
                entry['source_addresses'] = self[
                    'wallet'].getAddressesByAccount(entry['account'])

            entry['wallet'] = self['wallet']

            coin_transaction = CoinTransaction(entry)
            transactions.append(coin_transaction)

        # sort result
        transactions = sorted(transactions,
                              key=lambda transaction: transaction[orderby],
                              reverse=reverse)

        # cache the result
        self._cache.store('transactions', cache_hash, transactions)
        return transactions
Exemple #42
0
class ErrorHandler:
    def __init__(self, api):
        self.api = api
        self.cache = Cacher()

    def api_test(self):
        """
        This instructs user to rerun program in a few minutes if rate limit has exceeded.
        If rate limit has exceeded before running program, sleep_on_rate_limit won't work properly.
        """
        try:
            _ = self.api.GetUser(user_id=813286)
        except twitter.error.TwitterError as ex:
            if ex.message[0]['code'] == 88:
                sys.exit(
                    'Rate limit for API calls has exceeded. Rerun program in a few minutes.'
                )
            else:
                raise ex

    def handle_50(self, arg_dict):
        """
        Error 50 means that the user is not found.
        :param arg_dict: All the arguments the error handling will need. Here, we only need user_id.
        """
        self.cache.remove_all_occurrences(user_id=arg_dict['user_id'])
        print(
            f"Requested user not found. Removed all occurrences of {arg_dict['user_id']} in cache"
        )

    def handle_63(self, arg_dict):
        """
        Error 63 means that the requested user has been suspended. We simply skip the user.
        :param arg_dict: All the arguments the error handling will need. Here, we only need user_id.
        """
        self.cache.remove_all_occurrences(user_id=arg_dict['user_id'])
        print(
            f"Requested user suspended. Removed all occurrences of {arg_dict['user_id']} in cache"
        )

    def handle_88(self):
        """
        Error 88 means that the Twitter API rate limit has been exceeded. We keep attempting a connection until it
        resolves. This bit of code should not be executed, as we specified sleep_on_rate_limit in the API constructor
        and tested the connection before hand with self.api_test().
        """
        while True:
            try:
                _ = self.api.GetUser(user_id=813286)
            except twitter.error.TwitterError as ex:
                if ex.message[0]['code'] == 88:
                    print('Waiting for rate limit to reset')
                    sleep(60)
                    continue
                else:
                    raise ex
            else:
                break

    def handle(self, ex, **kwargs):
        """
        This acts as a switchboard for each handling method. Note how all the error handlers' names follow the style
        self.handle_{code}().
        :param ex: The raised exception. We need it to identify the error code raised.
        :param kwargs: The specific arguments we need in order to fix the error.
        """
        code = ex.message[0]['code']
        try:
            _ = getattr(self, 'handle_' + str(code))(kwargs)
        except AttributeError:
            print('No handler has been written for this error. Whoops.')
            raise ex
Exemple #43
0
class Extractor(object):
    
    def __init__(self):
        self._cache_enabled = False
        self._cache_connection_port = None
        self._cache_connection_host = None
        self._cacher = None
        self._arg = {"country_codes" : ["hrv", "usa"],
               "indicator_codes" : ["SP.POP.TOTL"],
               "start_date" : 1980,
               "end_date" : 2010,
               "pause" : 0}

    def arg(self):
        return copy.deepcopy(self._arg)
    
    def normalize(self, arg):
        if arg==None:
            arg = self.arg()
        if "interval" in arg:
            arg["start_date"], arg["end_date"] = arg["interval"]
        capitalize_list = lambda elems : [el.upper() for el in elems]
        arg["country_codes"] = capitalize_list(arg["country_codes"])
        return arg
    
    def grab(self, arg=None):
        rca_indicators = []
        wb_indicators = []
        
        try:
            import local_storage.rca.api
            # Check if indicator is in RCA dataset, if not then it must be in WB.
            # TODO: Make indicator type checking universal.
            available_rca_indicators = set(local_storage.rca.api.all_indicators()) 
            for indicator_code in arg["indicator_codes"]:
                if indicator_code in available_rca_indicators:
                    rca_indicators.append(indicator_code)
                else:
                    wb_indicators.append(indicator_code)
            
            # Grab RCA data.        
            rca_arg = copy.deepcopy(arg)
            rca_arg["indicator_codes"] = rca_indicators
            rca_countries = self._grab_from_api(rca_arg, local_storage.rca.api)
        except ImportError:
            print("Can't import local_storage.rca.api! Will assume there aren't any indicators from there")
            rca_countries = []
            wb_indicators = arg["indicator_codes"]
        
        # Grab WB data.
        wb_arg = copy.deepcopy(arg)
        wb_arg["indicator_codes"] = wb_indicators
        wb_countries = self._grab_from_api(wb_arg, wb.api)
        
        #TODO: Separate function for merging two or more list of countries.
        # Not assuming that wb and rca countries are in corresponding order in lists.
        all_codes = set()
        wb_dict = {}
        for wb_country in wb_countries:
            all_codes.add(wb_country.code)
            wb_dict[wb_country.code] = wb_country
        rca_dict = {}
        for rca_country in rca_countries:
            all_codes.add(rca_country.code)
            rca_dict[rca_country.code] = rca_country
        combined_countries = []
        for code in all_codes:
            current_countries = []
            try: current_countries.append(wb_dict[code])
            except KeyError: pass
            try: current_countries.append(rca_dict[code])
            except KeyError: pass
            if len(current_countries)!=0:
                if len(current_countries)>1:
                    current_countries[0].merge_with_country(current_countries[1])
                combined_countries.append(current_countries[0])
        return combined_countries
                
#        #TODO: Merge RCA and WB data.
#        # Assumptions is that rca_countries and wb_countries have elements in same order (maybe write unit test for that).
#        for wb_country,rca_country in zip(wb_countries,rca_countries):
#            wb_country.merge_with_country(rca_country)
#        
#        return wb_countries
    
    def _grab_from_api(self, arg=None, api=wb.api):
        #TODO: think about using kwargs
        """
        possible args are:
        
        1) the default by leaving None, defined in arg()
        2) setting your own such as:
                {"country_codes" : ["hrv", "usa", "chn"],
                   "indicator_codes" : ["SP.POP.TOTL"],
                   "start_date":1980,
                   "end_date":2010}
        3) using interval to set the dates more quickly (overrides start/end_date)
                {"country_codes" : ["hrv", "usa", "chn"],
                   "indicator_codes" : ["SP.POP.TOTL"],
                   "interval":(1980,2010)}
        """
        arg = self.normalize(arg)
        if self._cache_enabled:
            countries = self._cacher.retreive(arg)
            if countries: self.cache_hit = True
            else: self.cache_hit = False
        if not self._cache_enabled or not self.cache_hit:
            countries = api.query_multiple_data(
                                                   arg["country_codes"], arg["indicator_codes"],
                                                   arg["start_date"], arg["end_date"])
            if arg["pause"]:
                time.sleep(arg["pause"])
            if self._cache_enabled and not self.cache_hit:
                self._cache(countries)
        return countries
    
    def grab_metadata(self, what="countries", api = wb.api):
        """
        @param what: string of what to get
         - 'countries' - list of countries
         - 'indicators' - list of indicators
        """
        if what=="countries":
            return api.all_countries()
        elif what=="indicators":
            return api.all_indicators()
    
    def enable_cache(self, host="localhost", port=27017, test=False):
        """
        @param host: the hostname of the server where mongodb is running
        @param port: port mongodb is listening on
        @param test: True if you want to work on a db separate than the main cache
        """
        self._cache_connection_host = host
        self._cache_connection_port = port
        self._cache_enabled = True
        self._cacher = Cacher(self._cache_connection_host, self._cache_connection_port, test)
        
    def disable_cache(self):
        self._cache_enabled = False
        
    def clear_cache(self):
        """
        clear everything from the cache
        """
        self._cacher.clear()
    
    def _cache(self,countries):
        if self._cache_enabled:
            self._cacher.cache(countries)
        else: return
    
    def is_cached(self, arg):
        arg = self.normalize(arg)
        countries = self._cacher.retreive(arg)
        if countries: return True
        else: return False
    def was_cached(self):
        """
        tells if the data was found in the cache on the last grab
        """
        return self.cache_hit
Exemple #44
0
 def __init__(self, api):
     self.api = api
     self.cache = Cacher()
Exemple #45
0
class CoinWallet(object):
    '''
    Class for a wallet
    '''
    def __init__(self, wallet_config):
        self._errors = []
        self._config = {}
        self._cache = Cacher({
            'accounts': {},
            'transactions': {},
            'transactiondetails': {},
            'balances': {},
            'info': {},
        })

        if type(wallet_config) is dict:
            self._config = wallet_config

    @property
    def provider_id(self):
        '''
        Property for the provider id
        '''
        return self.get('id', None)

    @property
    def enabled(self):
        '''
        Return the status (enabled/disabled)
        '''
        return self.get('enabled', False)

    def __getitem__(self, key):
        '''
        Getter for dictionary-line behavior
        '''
        if key == "currency_symbol":
            return self.getCurrencySymbol()
        elif key == "currency_code":
            return self.getCurrencyCode()

        config = getattr(self, '_config')
        return config[key]

    def __setitem__(self, key, value):
        '''
        Setter for dictionary-line behavior
        '''
        config = getattr(self, '_config')
        config[key] = value
        return setattr(self, '_config', config)

    def get(self, key, default=False):
        '''
        Implementing .get() method for dictionary-line behavior
        '''
        if self._config.get(key, False):
            return self._config.get(key, False)
        else:
            return default

    def haskey(self, key):
        '''
        Check the existence of key
        '''
        if key in self._config.keys():
            return True
        else:
            return False

    def getNet(self):
        '''
        Return network value, mainnet or testnet
        '''
        # check for cached data, use that or get it again
        cache_hash = self.getParamHash("provider_id=%s" % self.provider_id)
        cached_peerinfo = self._cache.fetch('info', cache_hash)
        if cached_peerinfo:
            info = cached_peerinfo
        else:
            info = connector.getInfo(self.provider_id)
            self._cache.store('info', cache_hash, info)

        is_testnet = False
        if info.has_key('testnet'):
            is_testnet = info.get('testnet')
            if is_testnet is False:
                return "mainnet"
            elif is_testnet is True:
                return "testnet"
        else:
            # default to mainnet
            return "mainnet"

    def balance(self):
        '''
        Get wallet balance
        '''
        # check for cached data, use that or get it again
        cache_hash = self.getParamHash("balance")
        cached_object = self._cache.fetch('balance', cache_hash)
        if cached_object:
            return cached_object.get(self.provider_id, "-")

        balance = connector.getBalance(self.provider_id)

        # store result in cache
        self._cache.store('balance', cache_hash, balance)

        return misc.longNumber(balance.get(self.provider_id, "-"))

    def getParamHash(self, param=""):
        '''
        This function takes a string and calculates a sha224 hash out of it. 
        It is used to hash the input parameters of functions/method in order to 
        uniquely identify a cached result based  only on the input parameters of 
        the function/method call.
        '''
        cache_hash = hashlib.sha224(param).hexdigest()
        return cache_hash

    def listAccounts(self, gethidden=False, getarchived=False):
        '''
        Get a list of accounts. This method also supports filtering, fetches address for each account etc.
        '''
        # check for cached data, use that or get it again
        cache_hash = self.getParamHash("gethidden=%s&getarchived=%s" %
                                       (gethidden, getarchived))
        cached_object = self._cache.fetch('accounts', cache_hash)
        if cached_object:
            return cached_object

        # get data from the connector (xxxcoind)
        fresh_accounts = connector.listAccounts(
            gethidden=False,
            getarchived=False,
            selected_provider_id=self.provider_id)

        # get a list of archived address
        address_ignore_list = []
        if not getarchived:
            ignore_list = accountFilter.objects.filter(status=1)
            for ignored_account in ignore_list:
                address_ignore_list.append(
                    ignored_account.address.encode('ascii'))

        # get a list of hidden accounts
        address_hidden_list = []
        if not gethidden:
            hidden_list = accountFilter.objects.filter(status=2)
            for hidden_account in hidden_list:
                address_hidden_list.append(
                    hidden_account.address.encode('ascii'))

        accountObjects = []
        for account_name, account_balance in fresh_accounts.get(
                self.provider_id, {}).items():
            '''
            # check all addresses if they are in the archive list
            for ignored_address in address_ignore_list:
                if ignored_address in account_addresses:
                    del account_addresses[account_addresses.index(ignored_address)]
            
            # check all addresses if they are in the hidden list
            hidden_flag = False
            for hidden_address in address_hidden_list:
                if hidden_address in account_addresses:
                    hidden_flag = True
            '''

            accountObjects.append(
                CoinAccount({
                    'name': account_name,
                    'balance': account_balance,
                    'currency': self['currency'],
                    'provider_id': self.provider_id,
                    'wallet': self,
                }))

        # cache the result
        self._cache.store('accounts', cache_hash, accountObjects)
        return accountObjects

    def getCurrencySymbol(self):
        '''
        Return the Unicode currency symbol
        '''
        return self.get('symbol', None)

    def getCurrencyCode(self):
        '''
        Return the currency code
        '''
        return self.get('currency', "").lower()

    def listTransactions(self, limit=10, start=0):
        '''
        Return a list of transactions wallet-wide
        '''
        # check for cached data, use that or get it again
        cache_hash = self.getParamHash("limit=%s&start=%s" % (limit, start))
        cached_object = self._cache.fetch('transactions', cache_hash)
        if cached_object:
            return cached_object

        transactions = []
        transactions_dicts = connector.listTransactionsByAccount(
            "*", self.provider_id, limit, start)
        for transaction in transactions_dicts:
            transaction['wallet'] = self
            transaction['currency'] = self.getCurrencyCode()
            transaction['currency_symbol'] = self.getCurrencySymbol()
            transaction['provider_id'] = self.provider_id
            transactions.append(CoinTransaction(transaction))

        self._cache.store('transactions', cache_hash, transactions)
        return transactions

    def getAccountByName(self, name):
        '''
        Return CoinAccount() for name
        '''
        accounts = self.listAccounts(gethidden=True, getarchived=True)
        for account in accounts:
            if account['name'] == name:
                return account

        return None

    def getTransactionById(self, txid):
        '''
        Return a transaction by txid
        '''
        # check for cached data, use that or get it again
        cache_hash = self.getParamHash("txid=%s" % (txid))
        cached_object = self._cache.fetch('transactiondetails', cache_hash)
        if cached_object:
            return cached_object

        transaction_details = connector.getTransaction(txid, self.provider_id)
        transaction_details['currency'] = self.getCurrencyCode()
        transaction_details['wallet'] = self

        self._cache.store('transactiondetails', cache_hash,
                          transaction_details)
        return CoinTransaction(transaction_details)

    def getAddressesByAccount(self, account):
        '''
        Get a list of address for account
        '''
        addresses_list = connector.getAddressesByAccount(
            account, self.provider_id)
        coinaddresses = []
        for address in addresses_list:
            coinaddresses.append(CoinAddress(address, account))

        return coinaddresses

    def getDefaultAccount(self):
        '''
        Return the CoinAccount object for the default wallet account
        '''
        accounts = self.listAccounts(gethidden=True, getarchived=True)
        for account in accounts:
            if len(account['name']) == 0:
                return account
        else:
            return None

    def getAccountByAddress(self, address):
        '''
        Return account by address
        '''
        accounts = self.listAccounts(gethidden=True, getarchived=True)

        target_account = None
        for account in accounts:
            for account_address in account['addresses']:
                if str(address) == str(account_address):
                    target_account = account
                    target_account['currency'] = self.getCurrencyCode()
                    target_account['provider_id'] = self.provider_id
                    return target_account
        else:
            return None

    def getAccountByIdentifier(self, identifier):
        '''
        Get account by identifier
        '''
        list_of_accounts = self.listAccounts(gethidden=True, getarchived=True)
        for account in list_of_accounts:
            if account.getIdentifier() == identifier:
                return account
        else:
            return None
 def __init__(self, api, search_radius=1000, auto_next=False):
     self.api = api
     self.search_radius = search_radius
     self.auto_next = auto_next
     self.cache = Cacher('cache2.txt')
#
# Read and analyze output from ptrace-sampler
#

import sys
import os
import time
import subprocess
import select
import re
import resource

from lib_finder import LibFinder
from cacher import Cacher

cache = Cacher()


class Mappings:
    def __init__(self):
        self.origLines = []
        self.mappings = []

    def parseLine(self, line):
        self.origLines.append(line)

        # b74b3000-b760f000 r-xp 00000000 09:00 9593032    /lib/tls/i686/cmov/libc-2.9.so
        #(addrs, perms, offset, dev, inode, path) = line.split()
        parts = line.split()

        addrs = parts[0].split('-')
class BidirectionalBFS:
    def __init__(self, api, search_radius=1000, auto_next=False):
        self.api = api
        self.search_radius = search_radius
        self.auto_next = auto_next
        self.cache = Cacher('cache2.txt')

    def origin_following_bfs(self, start_user):
        """
        If the origin follows user A and B, and user B follows C, a graph can be drown as such:
        Origin ---> A
               ---> B ----> C
        Every yield expands the radius of the search by 1, until it hits search_radius
        :param start_user: The user from which to start the search
        :yield: All paths possible up to the current radius, in format set({user_id: path_to_that_user_id}, etc.)
        """
        api = self.api
        error_handler = ErrorHandler(api=api)
        queue = Queue()
        queue.put(start_user)
        visited = {start_user}
        path_to = defaultdict(lambda: [start_user])
        _ = path_to[start_user]

        for current_radius in range(self.search_radius):
            if queue.qsize() == 0:
                print('Queue empty. Ending origin BFS.')
                return
            current_user = queue.get()

            if self.cache.read_from_cache(
                    user_id=current_user
            ) is None:  # current_user does not yet exist in cache
                current_user_followings = api.GetFriendIDs(
                    user_id=current_user)
                current_user_screen_name = api.GetUser(
                    user_id=current_user).name
                self.cache.append_to_cache(
                    user_id=current_user,
                    screen_name=current_user_screen_name,
                    following_ids=current_user_followings)
                print(
                    f"Origin BFS: requested {current_user_screen_name}'s screen name and followings from API. "
                    f"Commencing search.")
            else:
                cached_data = self.cache.read_from_cache(user_id=current_user)
                current_user_screen_name = cached_data[
                    'screen_name']  # if user_id exists as a key in cache,
                # screen_name will always exist as well
                if not cached_data[
                        'following']:  # i.e. following list does not yet exist in cache
                    current_user_followings = api.GetFriendIDs(
                        user_id=current_user)
                    self.cache.append_to_cache(
                        user_id=current_user,
                        screen_name=current_user_screen_name,
                        following_ids=current_user_followings)
                    print(
                        f"Origin BFS: retrieved {current_user_screen_name}'s screen name from cache. "
                        f"Requested followings from API. Commencing search.")
                else:
                    current_user_followings = cached_data['following']
                    print(
                        f"Origin BFS: retrieved {current_user_screen_name}'s screen name AND followings from cache. "
                        f"Today is a good day. Commencing search.")

            num_followings = len(current_user_followings)
            for num, following in enumerate(current_user_followings):
                if following not in visited:
                    try:
                        if self.cache.read_from_cache(
                                user_id=following
                        ) is None:  # this user does not yet exist in cache
                            following_screen_name = api.GetUser(
                                user_id=following).name
                            self.cache.append_to_cache(
                                user_id=following,
                                screen_name=following_screen_name)
                            print(
                                f"Origin BFS: {current_user_screen_name} follows {following_screen_name}. "
                                f"Requested {following_screen_name}'s screen name from API and cached."
                            )
                        else:
                            following_screen_name = self.cache.read_from_cache(
                                user_id=following)['screen_name']
                            print(
                                f"Origin BFS: {current_user_screen_name} follows {following_screen_name}. "
                                f"Retrieved {following_screen_name}'s screen name from cache."
                            )

                        visited.add(following)
                        queue.put(following)
                        current_path = list(path_to[current_user])
                        current_path.extend([following])
                        path_to[following] = current_path
                    except twitter.error.TwitterError as ex:
                        error_handler.handle(ex, user_id=following)
                else:
                    print('User already visited. Skipping...')
                print(num + 1, '/', num_followings)
            yield path_to

    def target_follower_bfs(self, start_user):
        """
        Same as origin_follower_bfs, but the other way round.
        If user A and user B follows target, and user C follows user B:
        C ---> B ---> target
               A --->
        :param start_user: This refers to the target user. The user from which to start the search.
        :yield: All paths possible up to the current radius, in format set({user_id: path_to_that_user_id}, etc.)
        """
        api = self.api
        error_handler = ErrorHandler(api=api)
        queue = Queue()
        queue.put(start_user)
        visited = {start_user}
        path_to = defaultdict(lambda: [start_user])
        _ = path_to[start_user]

        for current_radius in range(self.search_radius):
            if queue.qsize() == 0:
                print('Queue empty. Ending target BFS.')
                return
            current_user = queue.get()

            if self.cache.read_from_cache(
                    user_id=current_user
            ) is None:  # current_user does not yet exist in cache
                current_user_followers = api.GetFollowerIDs(
                    user_id=current_user)
                current_user_screen_name = api.GetUser(
                    user_id=current_user).name
                self.cache.append_to_cache(
                    user_id=current_user,
                    screen_name=current_user_screen_name,
                    follower_ids=current_user_followers)
                print(
                    f"Target BFS: requested {current_user_screen_name}'s screen name and followers from API. "
                    f"Commencing search.")
            else:
                cached_data = self.cache.read_from_cache(user_id=current_user)
                current_user_screen_name = cached_data[
                    'screen_name']  # if user_id exists as a key in cache,
                # screen_name will always exist as well
                if not cached_data[
                        'followers']:  # i.e. follower list does not yet exist in cache
                    current_user_followers = api.GetFollowerIDs(
                        user_id=current_user)
                    self.cache.append_to_cache(
                        user_id=current_user,
                        screen_name=current_user_screen_name,
                        follower_ids=current_user_followers)
                    print(
                        f"Target BFS: retrieved {current_user_screen_name}'s screen name from cache. "
                        f"Requested followers from API. Commencing search.")
                else:
                    current_user_followers = cached_data['followers']
                    print(
                        f"Target BFS: retrieved {current_user_screen_name}'s screen name AND followers from cache. "
                        f"Today is a good day. Commencing search.")

            num_followers = len(current_user_followers)
            for num, follower in enumerate(current_user_followers):
                if follower not in visited:
                    try:
                        if self.cache.read_from_cache(
                                user_id=follower
                        ) is None:  # this user does not yet exist in cache
                            follower_screen_name = api.GetUser(
                                user_id=follower).name
                            self.cache.append_to_cache(
                                user_id=follower,
                                screen_name=follower_screen_name)
                            print(
                                f"Target BFS: {current_user_screen_name} is followed by {follower_screen_name}. "
                                f"Requested {follower_screen_name}'s screen name from API and cached."
                            )
                        else:
                            follower_screen_name = self.cache.read_from_cache(
                                user_id=follower)['screen_name']
                            print(
                                f"Target BFS: {current_user_screen_name} is followed by {follower_screen_name}. "
                                f"Retrieved {follower_screen_name}'s screen name from cache."
                            )

                        visited.add(follower)
                        queue.put(follower)
                        current_path = list(path_to[current_user])
                        current_path.extend([follower])
                        path_to[follower] = current_path
                    except twitter.error.TwitterError as ex:
                        error_handler.handle(ex, user_id=follower)
                else:
                    print('User already visited. Skipping...')
                print(num + 1, '/', num_followers)
            yield path_to

    def find_middle_users(self, paths_left, paths_right):
        """
        Find intersection between two paths, i.e. the 'middle users'
        :param paths_left:
        :param paths_right:
        :return: List of middle users
        """
        middle_users = paths_left.keys() & paths_right.keys()
        return middle_users

    def reduce_path(self, paths_left, paths_right, middle_user):
        """
        Reduce path from path from the left to path to the right, given the middle element
        :param paths_left:
        :param paths_right:
        :param middle_user:
        :return: Reduced paths
        """
        possible_paths = []

        for middle_user in middle_user:
            new_path = paths_left[middle_user]
            new_path.extend(list(reversed(paths_right[middle_user]))[1:])
            possible_paths.append(new_path)
        return possible_paths

    def display_paths(self, paths):
        """
        Simple UI method that prints the path out in English
        :param paths: list of list of paths. will print them all.
        """
        for path in paths:
            for idx, user in enumerate(path):
                screen_name = self.cache.read_from_cache(
                    user_id=user)['screen_name']
                if idx == 0:
                    print(f'{screen_name} follows ', end='')
                elif idx == len(path) - 1:
                    print(screen_name)
                else:
                    print(f'{screen_name}, who follows ', end='')

    def run_search(self, origin_user, target_user):
        """
        Driver code for the whole class.
        :param origin_user: The user to find a connection from
        :param target_user: The user to find a connection to
        :return: Possible paths in a list of lists of Twitter IDs, in X following Y order
        """
        for idx, (paths_left, paths_right) in enumerate(
                zip(self.origin_following_bfs(start_user=origin_user),
                    self.target_follower_bfs(start_user=target_user))):
            print(
                f"{len(paths_left)} total path(s) from origin. {len(paths_right)} total path(s) from target."
            )
            middle_users = self.find_middle_users(paths_left, paths_right)
            if len(middle_users) != 0:
                possible_paths = self.reduce_path(paths_left, paths_right,
                                                  middle_users)
                print(f"{len(possible_paths)} possible path(s) found:")
                self.display_paths(possible_paths)
            else:
                print("No possible paths found.")
            if idx == self.search_radius - 1:
                input(
                    "Search concluded with set search_radius. Press enter to close."
                )
                return possible_paths
            if not self.auto_next:
                input("Press enter to go to next radius")