def _run_map(self):
args = self.args
# if doing anything online, revalidate cache
Cacher.revalidate(args.map_id, self.users_info)
if (args.map_id and args.user_id): # passed both -m and -u but not -l
replays2 = [
OnlineReplay.from_map(args.map_id, user_id, args.cache,
replay_id)
for user_id, replay_id in self.users_info.items()
]
comparer = Comparer(args.threshold,
self.replays_check,
replays2=replays2)
comparer.compare(mode="double")
return
if (args.map_id): # only passed -m
# get all 50 top replays
replays = [
OnlineReplay.from_map(args.map_id, user_id, args.cache,
replay_id)
for user_id, replay_id in self.users_info.items()
]
comparer = Comparer(args.threshold, replays)
comparer.compare(mode="single")
return
def __init__(self, args):
"""
Initializes a Circleguard instance.
[SimpleNamespace or argparse.Namespace] args:
A namespace-like object representing how and what to compare. An example may look like
`Namespace(cache=False, local=False, map_id=None, number=50, threshold=20, user_id=None)`
"""
# get all replays in path to check against. Load this per circleguard instance or users moving files around while the gui is open doesn't work.
self.PATH_REPLAYS = [
join(PATH_REPLAYS_STUB, f) for f in os.listdir(PATH_REPLAYS_STUB)
if isfile(join(PATH_REPLAYS_STUB, f)) and f != ".DS_Store"
]
self.cacher = Cacher(args.cache)
self.args = args
if (args.map_id):
self.users_info = Loader.users_info(args.map_id, args.number)
if (args.user_id and args.map_id):
user_info = Loader.user_info(
args.map_id, args.user_id
)[args.
user_id] # should be guaranteed to only be a single mapping of user_id to a list
self.replays_check = [
OnlineReplay.from_map(self.cacher, args.map_id, args.user_id,
user_info[0], user_info[1], user_info[2])
]
class WebStash:
def __init__(self, getterType='urlopen', waitTimeBeforeScraping=0):
self.cacher = Cacher()
self.config = Config()
self.getter = Getter(getterType,
waitTimeBeforeScraping=waitTimeBeforeScraping)
def get_web_data(self, url):
try:
return self.cacher[url]
except KeyError:
self.config.debugPrint('Getting webData...')
filename = self.cacher.getFilename(url)
html = self.getter.get_html(url)
screenshotLocation = self.getter.get_screenshot(
url, filename + '.png')
webData = WebData(filename,
url,
html,
screenshotLocation=screenshotLocation)
self.cacher[url] = webData
return self.cacher[url]
def delete(url):
del self.cacher[url]
def clean(self):
self.cacher.clean()
async def download_file(url, filename, cacher: Cacher, useragent=''):
print(url)
async with aiohttp.request('get', url, headers={
'user-agent': useragent,
}) as res:
content = await res.read()
cacher.set(filename, content)
def __init__(self, reporter, waiter, outdir, useragent) -> None:
super(Keiba, self).__init__()
self.reporter: Reporter = reporter
self.waiter = waiter
self.outdir = outdir
self.useragent = useragent
self.cacher = Cacher(self.outdir)
self.semaphore = Semaphore(2)
def enable_cache(self, host="localhost", port=27017, test=False):
"""
@param host: the hostname of the server where mongodb is running
@param port: port mongodb is listening on
@param test: True if you want to work on a db separate than the main cache
"""
self._cache_connection_host = host
self._cache_connection_port = port
self._cache_enabled = True
self._cacher = Cacher(self._cache_connection_host,
self._cache_connection_port, test)
class Anicobin(Collector):
def __init__(self, reporter, waiter, outdir, useragent) -> None:
super(Anicobin, self).__init__()
self.reporter: Reporter = reporter
self.waiter = waiter
self.outdir = outdir
self.useragent = useragent
self.cacher = Cacher(self.outdir)
self.semaphore = Semaphore(2)
async def get(self, url):
filename = urllib.parse.quote(url, safe='') + '.html'
cache, _ = self.cacher.get(filename)
if cache:
html = cache
else:
await self.waiter.wait(url)
print('fetching', url)
async with aiohttp.request('get',
url,
headers={'user-agent':
self.useragent}) as req:
content = await req.read()
html = content.decode(SITE_ENCODING)
self.cacher.set(filename, html)
return html
async def collect(self, base_url, queue_size=3):
async def f(page):
print(page)
html, _ = await self.async_retry(3, self.get,
f'{base_url}?p={page}')
result = []
for post_url in await self.run_in_executor(get_post_urls, html):
_html = await self.get(post_url)
urls = get_pict_urls(_html)
for url in urls:
filename = urllib.parse.quote(url, safe='')
content, _ = self.cacher.get(filename, binary=True)
if not content:
await self.add_future(
'dlimage', download_file(url, filename,
self.cacher))
result.extend(urls)
return len(result) > 0
await self.queued_paging(1,
1000,
lambda page: f(page),
queue_size=queue_size)
def __init__(self, wallet_config):
self._errors = []
self._config = {}
self._cache = Cacher({
'accounts': {},
'transactions': {},
'transactiondetails': {},
'balances': {},
'info': {},
})
if type(wallet_config) is dict:
self._config = wallet_config
def __init__(self, accountDetails):
self._errors = []
self._account = {}
self._hidden = False
self._cache = Cacher({
'transactions': {},
'balances': {},
'addressesbyaccount': {},
})
if type(accountDetails) is dict:
self._account = accountDetails
self._provider_id = accountDetails['provider_id']
def default_request( self, req ):
target = 'z_robot_web_tools'
m = re.match( '/jenkins/(.*)', req.path_info )
if m:
target = m.group( 1 )
cacher = Cacher( cacheConfig )
data = cacher.deliver( target, mapFnc=self.cacher_deliver_mapFnc )
data[ 'a_refresh' ] = \
html.A( 'Refresh now', href=req.href.jenkins( 'refresh' ))
data[ 'tabs' ] = self._tab_data
data[ 'target' ] = target
data[ 'tabs_base' ] = req.base_path + '/jenkins'
return 'jenkins.html', data, None
def __init__(self,
reporter: Reporter,
waiter: Waiter,
outdir: str,
useragent: str = ''):
super(WearCollector, self).__init__()
self.reporter: Reporter = reporter
self.waiter = waiter
self.outdir = outdir
self.useragent = useragent
self.cacher = Cacher(self.outdir)
# 非同期処理の同時接続数制御
self.semaphore = Semaphore(2)
# ファイルダウンローダ
self.downloader = Downloader(self.waiter, self.semaphore,
self.reporter)
def __init__(self, args):
"""
Initializes an Anticheat instance.
[SimpleNamespace or argparse.Namespace] args:
A namespace-like object representing how and what to compare. An example may look like
`Namespace(cache=False, local=False, map_id=None, number=50, threshold=20, user_id=None)`
"""
self.cacher = Cacher(args.cache)
self.args = args
if(args.map_id):
self.users_info = Loader.users_info(args.map_id, args.number)
if(args.user_id and args.map_id):
user_info = Loader.user_info(args.map_id, args.user_id)
self.replays_check = [OnlineReplay.from_map(self.cacher, args.map_id, args.user_id, user_info[args.user_id][0], user_info[args.user_id][1])]
def __init__(self, transactionDetails):
self._transaction = {}
self._cache = Cacher({
'details': {},
})
if type(transactionDetails) is dict:
self._transaction = transactionDetails
self['timereceived_pretty'] = misc.twitterizeDate(
self.get('timereceived', 'never'))
self['time_pretty'] = misc.twitterizeDate(self.get(
'time', 'never'))
self['timereceived_human'] = datetime.datetime.fromtimestamp(
self.get('timereceived', 0))
self['time_human'] = datetime.datetime.fromtimestamp(
self.get('time', 0))
self['blocktime_human'] = datetime.datetime.fromtimestamp(
self.get('blocktime', 0))
self['blocktime_pretty'] = misc.twitterizeDate(
self.get('blocktime', 'never'))
self['currency_symbol'] = misc.getCurrencySymbol(
connector, self['currency'])
if self.get('category', False) in ['receive', 'send']:
if self['confirmations'] <= MainConfig['globals'][
'confirmation_limit']:
self['status_icon'] = 'glyphicon-time'
self['status_color'] = '#AAA'
self['tooltip'] = self['confirmations']
else:
self['status_icon'] = 'glyphicon-ok-circle'
self['status_color'] = '#1C9E3F'
self['tooltip'] = self['confirmations']
accountObject = self['wallet'].getAccountByName(self['account'])
self['account'] = accountObject
if self['category'] == 'receive':
self['icon'] = 'glyphicon-circle-arrow-down'
elif self['category'] == 'send':
self['icon'] = 'glyphicon-circle-arrow-up'
elif self['category'] == 'move':
self['icon'] = 'glyphicon-circle-arrow-right'
self['otheraccount'] = self['wallet'].getAccountByName(
self['otheraccount'])
def wrapper(*args, **kwargs):
map_id = args[0]
user_id = args[1]
lzma = Cacher.check_cache(map_id, user_id)
if (lzma):
replay_data = osrparse.parse_replay(lzma, pure_lzma=True).play_data
return Replay(replay_data, user_id)
else:
return function(*args, **kwargs)
def from_map(map_id, user_id, cache, replay_id):
"""
Creates a Replay instance from a replay by the given user on the given map.
Args:
String map_id: The map_id to download the replay from.
String user_id: The user id to download the replay of.
Also used as the username of the Replay.
Returns:
The Replay instance created with the given information.
"""
lzma_bytes = Loader.replay_data(map_id, user_id)
parsed_replay = osrparse.parse_replay(lzma_bytes, pure_lzma=True)
replay_data = parsed_replay.play_data
if (cache):
Cacher.cache(map_id, user_id, lzma_bytes, replay_id)
return OnlineReplay(replay_data, user_id)
def enable_cache(self, host="localhost", port=27017, test=False):
"""
@param host: the hostname of the server where mongodb is running
@param port: port mongodb is listening on
@param test: True if you want to work on a db separate than the main cache
"""
self._cache_connection_host = host
self._cache_connection_port = port
self._cache_enabled = True
self._cacher = Cacher(self._cache_connection_host, self._cache_connection_port, test)
def __init__(self, wallet_config):
self._errors = []
self._config = {}
self._cache = Cacher({
'accounts': {},
'transactions': {},
'transactiondetails': {},
'balances': {},
'info': {},
})
if type(wallet_config) is dict:
self._config = wallet_config
def setup():
try:
shutil.rmtree('webstashcache')
except FileNotFoundError:
pass
try:
os.remove('cacheMap.pkl')
except FileNotFoundError:
pass
cacher = Cacher()
link = 'https://news.ycombinator.com/news'
return (cacher, link)
def __init__(self, accountDetails):
self._errors = []
self._account = {}
self._hidden = False
self._cache = Cacher({
'transactions': {},
'balances': {},
'addressesbyaccount': {},
})
if type(accountDetails) is dict:
self._account = accountDetails
self._provider_id = accountDetails['provider_id']
def get_element_processor(element, send_cache, verbose, dryrun):
"""Gets an instance of element that contains cached data, depending on
arguments passed.
Will skip any cached data if send_cache is False or dryrun is True.
@param element: Element class that should be fetched.
@type element: L{MetaElement} sub class.
@param send_cache: Indicates whether cache should be included.
@type send_cache: bool
@param verbose: Indicates whether or not the script runs in verbose mode.
@type verbose: bool
@param dryrun: Indicates whether the script is doing a dry run.
@type dryrun: bool
@return: L{MetaElement} sub class. The sub class is given by the element
parameter.
"""
if dryrun or not send_cache:
return element()
if send_cache:
c = Cacher(element.xml_tag_name)
cached_data = c.get_cache()
if cached_data is not None:
if element.resend_cache:
element_processor = element.from_xml_element(cached_data,
element)
else:
logging.info(("Found cached data for \"%s\", but element "
"type declares not to resend this cache. "
"Removing cache.") % element.xml_tag_name)
c.remove_cache()
element_processor = element()
if element_processor is None:
logging.error(("Found cached data for \"%s\", but unable to "
"load. Check file \"%s\" for possible errors. "
"Continuing without cached data") %
(element.xml_tag_name, c.file_path))
element_processor = element()
else:
# We have successfully loaded the cached data.
if element.resend_cache:
logging.debug(("Succesfully loaded cache for \"%s\", "
"removing cached file \"%s\".") %
(element.xml_tag_name, c.file_path))
c.remove_cache()
else:
logging.debug(("Found no cached data for \"%s\".") %
element.xml_tag_name)
element_processor = element()
return element_processor
def __init__(self, transactionDetails):
self._transaction = {}
self. _cache = Cacher({
'details': {},
})
if type(transactionDetails) is dict:
self._transaction = transactionDetails
self['timereceived_pretty'] = misc.twitterizeDate(self.get('timereceived', 'never'))
self['time_pretty'] = misc.twitterizeDate(self.get('time', 'never'))
self['timereceived_human'] = datetime.datetime.fromtimestamp(self.get('timereceived', 0))
self['time_human'] = datetime.datetime.fromtimestamp(self.get('time', 0))
self['blocktime_human'] = datetime.datetime.fromtimestamp(self.get('blocktime', 0))
self['blocktime_pretty'] = misc.twitterizeDate(self.get('blocktime', 'never'))
self['currency_symbol'] = misc.getCurrencySymbol(connector, self['currency'])
if self.get('category', False) in ['receive', 'send']:
if self['confirmations'] <= MainConfig['globals']['confirmation_limit']:
self['status_icon'] = 'glyphicon-time'
self['status_color'] = '#AAA';
self['tooltip'] = self['confirmations']
else:
self['status_icon'] = 'glyphicon-ok-circle'
self['status_color'] = '#1C9E3F';
self['tooltip'] = self['confirmations']
accountObject = self['wallet'].getAccountByName(self['account'])
self['account'] = accountObject
if self['category'] == 'receive':
self['icon'] = 'glyphicon-circle-arrow-down'
elif self['category'] == 'send':
self['icon'] = 'glyphicon-circle-arrow-up'
elif self['category'] == 'move':
self['icon'] = 'glyphicon-circle-arrow-right'
self['otheraccount'] = self['wallet'].getAccountByName(self['otheraccount'])
def refresh_request( self, req ):
cacher = Cacher( self._tab_data )
cacher.cache()
success = True
return 'refresh.html', { 'success': success }, None
class CoinTransaction(object):
'''
Class for a transaction
'''
# source: https://github.com/zamgo/PHPCoinAddress/blob/master/README.md
prefixes = {
'btc': {
'mainnet': '\x00',
'testnet': '\x6f'
},
'ltc': {
'mainnet': '\x30',
'testnet': '\x6f'
},
'ftc': {
'mainnet': '\x0E',
'testnet': '\x6f'
},
'ppc': {
'mainnet': '\x37',
'testnet': '\x6f'
},
'nmc': {
'mainnet': '\x34',
'testnet': '\x6f'
},
'nvc': {
'mainnet': '\x08',
'testnet': '\x6f'
},
'doge': {
'mainnet': '\x30',
'testnet': '\x6f'
},
}
def __init__(self, transactionDetails):
self._transaction = {}
self._cache = Cacher({
'details': {},
})
if type(transactionDetails) is dict:
self._transaction = transactionDetails
self['timereceived_pretty'] = misc.twitterizeDate(
self.get('timereceived', 'never'))
self['time_pretty'] = misc.twitterizeDate(self.get(
'time', 'never'))
self['timereceived_human'] = datetime.datetime.fromtimestamp(
self.get('timereceived', 0))
self['time_human'] = datetime.datetime.fromtimestamp(
self.get('time', 0))
self['blocktime_human'] = datetime.datetime.fromtimestamp(
self.get('blocktime', 0))
self['blocktime_pretty'] = misc.twitterizeDate(
self.get('blocktime', 'never'))
self['currency_symbol'] = misc.getCurrencySymbol(
connector, self['currency'])
if self.get('category', False) in ['receive', 'send']:
if self['confirmations'] <= MainConfig['globals'][
'confirmation_limit']:
self['status_icon'] = 'glyphicon-time'
self['status_color'] = '#AAA'
self['tooltip'] = self['confirmations']
else:
self['status_icon'] = 'glyphicon-ok-circle'
self['status_color'] = '#1C9E3F'
self['tooltip'] = self['confirmations']
accountObject = self['wallet'].getAccountByName(self['account'])
self['account'] = accountObject
if self['category'] == 'receive':
self['icon'] = 'glyphicon-circle-arrow-down'
elif self['category'] == 'send':
self['icon'] = 'glyphicon-circle-arrow-up'
elif self['category'] == 'move':
self['icon'] = 'glyphicon-circle-arrow-right'
self['otheraccount'] = self['wallet'].getAccountByName(
self['otheraccount'])
def __getitem__(self, key):
'''
Getter for dictionary-line behavior
'''
if key == "account":
if self.haskey("account"):
return self._transaction['account']
else:
return self._transaction['details'][0]['account']
elif key == "category":
if self.haskey("category"):
return self._transaction['category']
else:
return self._transaction['details'][0]['category']
elif key == "currency_symbol":
return self.getCurrencySymbol()
elif key == "currency_code":
return self.getCurrencyCode()
elif key == "raw_transaction":
return self.getRawTransaction()
elif key == "source_address":
return self.getSenderAddress()
elif key == "address":
return CoinAddress(self._transaction['address'],
self._transaction['account'])
transaction = getattr(self, '_transaction')
return transaction.get(key, None)
def __setitem__(self, key, value):
'''
Setter for dictionary-line behavior
'''
transaction = getattr(self, '_transaction')
transaction[key] = value
return setattr(self, '_transaction', transaction)
def get(self, key, default=False):
'''
get() method for dictionary-line behavior
'''
if self._transaction.get(key, False):
return self._transaction.get(key, False)
else:
return default
def haskey(self, key):
'''
Check the existence of key
'''
if key in self._transaction.keys():
return True
else:
return False
@property
def provider_id(self):
'''
Property for the provider id
'''
return self.get('provider_id', None)
@property
def transaction_id(self):
'''
Property for the txid
'''
return self.get('txid', None)
@property
def txid(self):
'''
Property for the txid
'''
return self.get('txid', None)
def getParamHash(self, param=""):
'''
This function takes a string and calculates a sha224 hash out of it.
It is used to hash the input parameters of functions/method in order to
uniquely identify a cached result based only on the input parameters of
the function/method call.
'''
cache_hash = hashlib.sha224(param).hexdigest()
return cache_hash
def metaProperties(self):
'''
Return transaction details, like sender address
'''
raw_transaction = self.getRawTransaction()
if self['category'] == 'receive':
sender_address = self.decodeScriptSig(raw_transaction,
self['currency'],
self['wallet'].getNet())
else:
sender_address = None
return {'sender_address': sender_address}
def getRawTransaction(self):
'''
Get the raw transaction dict
'''
cache_hash = self.getParamHash("details")
cached_object = self._cache.fetch('details', cache_hash)
if cached_object:
raw_transaction = cached_object
else:
raw_transaction = connector.getRawTransaction(
self['txid'], self['wallet']['provider_id'])
self._cache.store('details', cache_hash, raw_transaction)
return raw_transaction
def getSenderAddress(self):
'''
Getter function for the sender address
'''
if self['category'] == 'receive':
meta_properties = self.metaProperties()
if meta_properties.get('sender_address', False):
return CoinAddress(meta_properties['sender_address'],
'This is a sender address!')
return None
def decodeScriptSig(self, rawtransaction, currency, net='testnet'):
'''
Decode input script signature, courtesy of:
http://bitcoin.stackexchange.com/questions/7838/why-does-gettransaction-report-me-only-the-receiving-address/8864#8864
'''
try:
script_sig = rawtransaction['vin'][0]['scriptSig']['asm']
except:
return "not enough info"
script = script_sig.split()
h = hashlib.sha256(script[1].decode("hex")).digest()
ripe160 = hashlib.new('ripemd160')
ripe160.update(h)
d = ripe160.digest()
prefix = self.prefixes[currency.lower()][net]
address = (prefix + d)
# calculate checksum
checksum = hashlib.sha256(
hashlib.sha256(address).digest()).digest()[:4]
# build the raw address
address += checksum
# encode the address in base58
encoded_address = misc.b58encode(address)
return encoded_address
def getCurrencySymbol(self):
'''
Return the Unicode currency symbol
'''
return misc.getCurrencySymbol(connector, self.getCurrencyCode())
def getCurrencyCode(self):
'''
Return the currency code
'''
return self.get('currency', "").lower()
def say_smarx_and_reset():
to_say = Cacher.get_message_by_user(slack_users.hes_zeebz, randomize=True)
Cacher.reset_messages_by_user(slack_users.hes_zeebz, randomize=True)
return to_say
def test_smarx(_):
return len(Cacher.get_messages_by_user(slack_users.hes_zeebz)) >= 25
class Anticheat:
def __init__(self, args):
"""
Initializes an Anticheat instance.
[SimpleNamespace or argparse.Namespace] args:
A namespace-like object representing how and what to compare. An example may look like
`Namespace(cache=False, local=False, map_id=None, number=50, threshold=20, user_id=None)`
"""
self.cacher = Cacher(args.cache)
self.args = args
if(args.map_id):
self.users_info = Loader.users_info(args.map_id, args.number)
if(args.user_id and args.map_id):
user_info = Loader.user_info(args.map_id, args.user_id)
self.replays_check = [OnlineReplay.from_map(self.cacher, args.map_id, args.user_id, user_info[args.user_id][0], user_info[args.user_id][1])]
def run(self):
"""
Starts loading and detecting replays based on the args passed through the command line.
"""
if(self.args.local):
self._run_local()
elif(self.args.map_id):
self._run_map()
else:
print("Please set either --local (-l) or --map (-m)! ")
sys.exit(1)
def _run_local(self):
args = self.args
# get all local user replays (used in every --local case)
replays1 = [LocalReplay.from_path(osr_path) for osr_path in PATH_REPLAYS_USER]
threshold = args.threshold
stddevs = args.stddevs
if(args.map_id and args.user_id):
# compare every local replay with just the given user + map replay
comparer = Comparer(threshold, args.silent, replays1, replays2=self.replays_check, stddevs=stddevs)
comparer.compare(mode="double")
return
if(args.map_id):
# compare every local replay with every leaderboard entry
replays2 = OnlineReplay.from_user_info(self.cacher, args.map_id, self.users_info)
comparer = Comparer(threshold, args.silent, replays1, replays2=replays2, stddevs=stddevs)
comparer.compare(mode="double")
return
if(args.single):
# checks every replay listed in PATH_REPLAYS_USER against every other replay there
comparer = Comparer(threshold, stddevs, args.silent, replays1)
comparer.compare(mode="single")
return
else:
# checks every replay listed in PATH_REPLAYS_USER against every replay listed in PATH_REPLAYS_CHECK
replays2 = [LocalReplay.from_path(osr_path) for osr_path in PATH_REPLAYS_CHECK]
comparer = Comparer(threshold, args.silent, replays1, replays2=replays2, stddevs=stddevs)
comparer.compare(mode="double")
return
def _run_map(self):
args = self.args
threshold = args.threshold
stddevs = args.stddevs
# if doing anything online, revalidate cache
self.cacher.revalidate(args.map_id, self.users_info)
if(args.map_id and args.user_id): # passed both -m and -u but not -l
replays2 = OnlineReplay.from_user_info(self.cacher, args.map_id, self.users_info)
comparer = Comparer(threshold, args.silent, self.replays_check, replays2=replays2, stddevs=stddevs)
comparer.compare(mode="double")
return
if(args.map_id): # only passed -m
# get all 50 top replays
replays = OnlineReplay.from_user_info(self.cacher, args.map_id, self.users_info)
comparer = Comparer(threshold, args.silent, replays, stddevs=stddevs)
comparer.compare(mode="single")
return
class CoinTransaction(object):
'''
Class for a transaction
'''
# source: https://github.com/zamgo/PHPCoinAddress/blob/master/README.md
prefixes = {
'btc': {'mainnet': '\x00', 'testnet': '\x6f'},
'ltc': {'mainnet': '\x30', 'testnet': '\x6f'},
'ftc': {'mainnet': '\x0E', 'testnet': '\x6f'},
'ppc': {'mainnet': '\x37', 'testnet': '\x6f'},
'nmc': {'mainnet': '\x34', 'testnet': '\x6f'},
'nvc': {'mainnet': '\x08', 'testnet': '\x6f'},
'doge': {'mainnet': '\x30', 'testnet': '\x6f'},
}
def __init__(self, transactionDetails):
self._transaction = {}
self. _cache = Cacher({
'details': {},
})
if type(transactionDetails) is dict:
self._transaction = transactionDetails
self['timereceived_pretty'] = misc.twitterizeDate(self.get('timereceived', 'never'))
self['time_pretty'] = misc.twitterizeDate(self.get('time', 'never'))
self['timereceived_human'] = datetime.datetime.fromtimestamp(self.get('timereceived', 0))
self['time_human'] = datetime.datetime.fromtimestamp(self.get('time', 0))
self['blocktime_human'] = datetime.datetime.fromtimestamp(self.get('blocktime', 0))
self['blocktime_pretty'] = misc.twitterizeDate(self.get('blocktime', 'never'))
self['currency_symbol'] = misc.getCurrencySymbol(connector, self['currency'])
if self.get('category', False) in ['receive', 'send']:
if self['confirmations'] <= MainConfig['globals']['confirmation_limit']:
self['status_icon'] = 'glyphicon-time'
self['status_color'] = '#AAA';
self['tooltip'] = self['confirmations']
else:
self['status_icon'] = 'glyphicon-ok-circle'
self['status_color'] = '#1C9E3F';
self['tooltip'] = self['confirmations']
accountObject = self['wallet'].getAccountByName(self['account'])
self['account'] = accountObject
if self['category'] == 'receive':
self['icon'] = 'glyphicon-circle-arrow-down'
elif self['category'] == 'send':
self['icon'] = 'glyphicon-circle-arrow-up'
elif self['category'] == 'move':
self['icon'] = 'glyphicon-circle-arrow-right'
self['otheraccount'] = self['wallet'].getAccountByName(self['otheraccount'])
def __getitem__(self, key):
'''
Getter for dictionary-line behavior
'''
if key == "account":
if self.haskey("account"):
return self._transaction['account']
else:
return self._transaction['details'][0]['account']
elif key == "category":
if self.haskey("category"):
return self._transaction['category']
else:
return self._transaction['details'][0]['category']
elif key == "currency_symbol":
return self.getCurrencySymbol()
elif key == "currency_code":
return self.getCurrencyCode()
elif key == "raw_transaction":
return self.getRawTransaction()
elif key == "source_address":
return self.getSenderAddress()
elif key == "address":
return CoinAddress(self._transaction['address'], self._transaction['account'])
transaction = getattr(self, '_transaction')
return transaction.get(key, None)
def __setitem__(self, key, value):
'''
Setter for dictionary-line behavior
'''
transaction = getattr(self, '_transaction')
transaction[key] = value
return setattr(self, '_transaction', transaction)
def get(self, key, default=False):
'''
get() method for dictionary-line behavior
'''
if self._transaction.get(key, False):
return self._transaction.get(key, False)
else:
return default
def haskey(self, key):
'''
Check the existence of key
'''
if key in self._transaction.keys():
return True
else:
return False
@property
def provider_id(self):
'''
Property for the provider id
'''
return self.get('provider_id', None)
@property
def transaction_id(self):
'''
Property for the txid
'''
return self.get('txid', None)
@property
def txid(self):
'''
Property for the txid
'''
return self.get('txid', None)
def getParamHash(self, param=""):
'''
This function takes a string and calculates a sha224 hash out of it.
It is used to hash the input parameters of functions/method in order to
uniquely identify a cached result based only on the input parameters of
the function/method call.
'''
cache_hash = hashlib.sha224(param).hexdigest()
return cache_hash
def metaProperties(self):
'''
Return transaction details, like sender address
'''
raw_transaction = self.getRawTransaction()
if self['category'] == 'receive':
sender_address = self.decodeScriptSig(raw_transaction, self['currency'], self['wallet'].getNet())
else:
sender_address = None
return {'sender_address': sender_address}
def getRawTransaction(self):
'''
Get the raw transaction dict
'''
cache_hash = self.getParamHash("details")
cached_object = self._cache.fetch('details', cache_hash)
if cached_object:
raw_transaction = cached_object
else:
raw_transaction = connector.getRawTransaction(self['txid'], self['wallet']['provider_id'])
self._cache.store('details', cache_hash, raw_transaction)
return raw_transaction
def getSenderAddress(self):
'''
Getter function for the sender address
'''
if self['category'] == 'receive':
meta_properties = self.metaProperties()
if meta_properties.get('sender_address', False):
return CoinAddress(meta_properties['sender_address'], 'This is a sender address!')
return None
def decodeScriptSig(self, rawtransaction, currency, net='testnet'):
'''
Decode input script signature, courtesy of:
http://bitcoin.stackexchange.com/questions/7838/why-does-gettransaction-report-me-only-the-receiving-address/8864#8864
'''
try:
script_sig = rawtransaction['vin'][0]['scriptSig']['asm']
except:
return "not enough info"
script = script_sig.split()
h = hashlib.sha256(script[1].decode("hex")).digest()
ripe160 = hashlib.new('ripemd160')
ripe160.update(h)
d = ripe160.digest()
prefix = self.prefixes[currency.lower()][net]
address = (prefix + d)
# calculate checksum
checksum = hashlib.sha256(hashlib.sha256(address).digest()).digest()[:4]
# build the raw address
address += checksum
# encode the address in base58
encoded_address = misc.b58encode(address)
return encoded_address
def getCurrencySymbol(self):
'''
Return the Unicode currency symbol
'''
return misc.getCurrencySymbol(connector, self.getCurrencyCode())
def getCurrencyCode(self):
'''
Return the currency code
'''
return self.get('currency', "").lower()
class Keiba(Collector):
def __init__(self, reporter, waiter, outdir, useragent) -> None:
super(Keiba, self).__init__()
self.reporter: Reporter = reporter
self.waiter = waiter
self.outdir = outdir
self.useragent = useragent
self.cacher = Cacher(self.outdir)
self.semaphore = Semaphore(2)
async def get_search_page(
self,
n: int,
options: dict = {
'pid': str,
'word': str,
'track[]': str,
'start_year': str,
'start_mon': str,
'end_year': str,
'end_mon': str,
'jyo[]': str,
'kyori_min': str,
'kyori_max': str,
'sort': str,
'list': str,
}):
url = 'https://db.netkeiba.com/'
psuedo_url = f'{url}?{urllib.parse.urlencode(options)}&page=1'
filename = urllib.parse.quote(psuedo_url + '.html', safe='')
cache, _ = self.cacher.get(filename)
if cache:
search_result = cache
else:
await self.waiter.wait(psuedo_url)
print('fetching', psuedo_url)
async with aiohttp.request(
'post',
url=url,
headers={
'content-type': 'application/x-www-form-urlencoded',
'user-agent': self.useragent
},
data=urllib.parse.urlencode(options)) as req:
content = await req.read()
search_result = content.decode(SITE_ENCODING)
if str(req.url) == url:
self.cacher.set(filename, search_result)
else:
print(f'Warning: redirected to {str(req.url)}')
self.cacher.set(
urllib.parse.quote(str(req.url), safe='') + '.html',
search_result)
return None
if n == 1:
return search_result
else:
data = await self.run_in_executor(get_nextpage_data, search_result)
data['page'] = str(n)
psuedo_url = f'{url}?{urllib.parse.urlencode(options)}&page={n}'
filename = urllib.parse.quote(psuedo_url + '.html', safe='')
cache, _ = self.cacher.get(filename)
if cache:
result = cache
else:
await self.waiter.wait(psuedo_url)
print('fetching', psuedo_url)
async with aiohttp.request(
'post',
url=url,
headers={
'content-type':
'application/x-www-form-urlencoded',
'user-agent': self.useragent
},
data=urllib.parse.urlencode(
data, encoding=SITE_ENCODING)) as req:
try:
content = await req.read()
result = content.decode(SITE_ENCODING)
self.cacher.set(filename, result)
except Exception as e:
print('get_tail', e)
return None
return result
async def get_race_page(self, url):
filename = urllib.parse.quote(url, safe='') + '.html'
cache, _ = self.cacher.get(filename, ext='')
if cache:
html = cache
else:
await self.waiter.wait(url)
print('fetching', url)
async with aiohttp.request('get',
url,
headers={'user-agent':
self.useragent}) as req:
content = await req.read()
html = content.decode(SITE_ENCODING)
self.cacher.set(filename, html)
return html
async def collect(self, year, queue_size=3):
async def f(page):
print(page)
html, _ = await self.async_retry(
3, self.get_search_page, page, {
'pid': 'race_list',
'start_year': str(year),
'end_year': str(year),
'sort': 'date',
'list': '100'
})
return len([
await self.add_future('get_race', self.get_race_page(race_url))
for race_url in await self.run_in_executor(
get_race_urls, html)
]) == 100 if html else False
await self.queued_paging(1,
1000,
lambda page: f(page),
queue_size=queue_size)
async def collect_horse(self, year, queue_size=3):
async def f(page):
print(page)
html, error = await self.async_retry(3, self.get_search_page, page,
{
'pid': 'horse_list',
'list': '100',
'birthyear': year,
})
if error:
print('Waringn: max retries exceeded')
return False
return len(await self.run_in_executor(
get_horse_urls, html)) == 100 if html else False
await self.queued_paging(1,
1000,
lambda page: f(page),
queue_size=queue_size)
def __init__(self, api):
self.api = api
self.cache = Cacher('cache2.txt')
class Circleguard:
def __init__(self, args):
"""
Initializes a Circleguard instance.
[SimpleNamespace or argparse.Namespace] args:
A namespace-like object representing how and what to compare. An example may look like
`Namespace(cache=False, local=False, map_id=None, number=50, threshold=20, user_id=None)`
"""
# get all replays in path to check against. Load this per circleguard instance or users moving files around while the gui is open doesn't work.
self.PATH_REPLAYS = [
join(PATH_REPLAYS_STUB, f) for f in os.listdir(PATH_REPLAYS_STUB)
if isfile(join(PATH_REPLAYS_STUB, f)) and f != ".DS_Store"
]
self.cacher = Cacher(args.cache)
self.args = args
if (args.map_id):
self.users_info = Loader.users_info(args.map_id, args.number)
if (args.user_id and args.map_id):
user_info = Loader.user_info(
args.map_id, args.user_id
)[args.
user_id] # should be guaranteed to only be a single mapping of user_id to a list
self.replays_check = [
OnlineReplay.from_map(self.cacher, args.map_id, args.user_id,
user_info[0], user_info[1], user_info[2])
]
def run(self):
"""
Starts loading and detecting replays based on the args passed through the command line.
"""
if (self.args.verify):
self._run_verify()
elif (self.args.local):
self._run_local()
elif (self.args.map_id):
self._run_map()
else:
print(
"Please set either --local (-l), --map (-m), or --verify (-v)! "
)
def _run_verify(self):
args = self.args
map_id = self.args.verify[0]
user1_id = self.args.verify[1]
user2_id = self.args.verify[2]
user1_info = Loader.user_info(map_id, user1_id)
user2_info = Loader.user_info(map_id, user2_id)
replay1 = OnlineReplay.from_user_info(self.cacher, map_id, user1_info)
replay2 = OnlineReplay.from_user_info(self.cacher, map_id, user2_info)
comparer = Comparer(args.threshold,
args.silent,
replay1,
replays2=replay2,
stddevs=args.stddevs)
comparer.compare(mode="double")
def _run_local(self):
args = self.args
# get all local replays (used in every --local case)
replays1 = [
LocalReplay.from_path(osr_path) for osr_path in self.PATH_REPLAYS
]
threshold = args.threshold
stddevs = args.stddevs
if (args.map_id and args.user_id):
# compare every local replay with just the given user + map replay
comparer = Comparer(threshold,
args.silent,
replays1,
replays2=self.replays_check,
stddevs=stddevs)
comparer.compare(mode="double")
return
if (args.map_id):
# compare every local replay with every leaderboard entry
replays2 = OnlineReplay.from_user_info(self.cacher, args.map_id,
self.users_info)
comparer = Comparer(threshold,
args.silent,
replays1,
replays2=replays2,
stddevs=stddevs)
comparer.compare(mode="double")
return
else:
comparer = Comparer(threshold,
args.silent,
replays1,
stddevs=stddevs)
comparer.compare(mode="single")
def _run_map(self):
args = self.args
threshold = args.threshold
stddevs = args.stddevs
# if doing anything online, revalidate cache
self.cacher.revalidate(args.map_id, self.users_info)
if (args.map_id and args.user_id): # passed both -m and -u but not -l
replays2 = OnlineReplay.from_user_info(self.cacher, args.map_id,
self.users_info)
comparer = Comparer(threshold,
args.silent,
self.replays_check,
replays2=replays2,
stddevs=stddevs)
comparer.compare(mode="double")
return
if (args.map_id): # only passed -m
# get all 50 top replays
replays = OnlineReplay.from_user_info(self.cacher, args.map_id,
self.users_info)
comparer = Comparer(threshold,
args.silent,
replays,
stddevs=stddevs)
comparer.compare(mode="single")
return
class NaiveBFS:
def __init__(self, api):
self.api = api
self.cache = Cacher('cache2.txt')
def bfs(self, start_user, target_user):
"""
Find shortest path from start_user to target_user, where a path is defined as:
X --> Y when user X follows user Y.
This is a horrible algorithm - don't use it unless you're willing to wait for around a week or longer
if you know the degree of separation is >= 2
:param start_user: The starting (origin) user
:param target_user: The user you're trying to find a path to
:return: A path from start_user to target_user
"""
api = self.api
error_handler = ErrorHandler(api=api)
queue = Queue()
queue.put(start_user)
visited = {start_user}
path_to = defaultdict(lambda: [start_user])
while queue.not_empty:
current_user = queue.get()
if self.cache.read_from_cache(
user_id=current_user
) is None: # current_user does not yet exist in cache
current_user_followings = api.GetFriendIDs(
user_id=current_user)
current_user_screen_name = api.GetUser(
user_id=current_user).name
self.cache.append_to_cache(
user_id=current_user,
screen_name=current_user_screen_name,
following_ids=current_user_followings)
print(
f"Naive BFS: requested {current_user_screen_name}'s screen name and followings from API. "
f"Commencing search.")
else:
cached_data = self.cache.read_from_cache(user_id=current_user)
current_user_screen_name = cached_data[
'screen_name'] # if user_id exists as a key in cache,
# screen_name will always exist as well
if not cached_data[
'following']: # i.e. following list does not yet exist in cache
current_user_followings = api.GetFriendIDs(
user_id=current_user)
self.cache.append_to_cache(
user_id=current_user,
screen_name=current_user_screen_name,
following_ids=current_user_followings)
print(
f"Naive BFS: retrieved {current_user_screen_name}'s screen name from cache. "
f"Requested followings from API. Commencing search.")
else:
current_user_followings = cached_data['following']
print(
f"Naive BFS: retrieved {current_user_screen_name}'s screen name AND followings from cache. "
f"Today is a good day. Commencing search.")
num_followings = len(current_user_followings)
for num, following in enumerate(current_user_followings):
if following not in visited:
try:
if self.cache.read_from_cache(
user_id=following
) is None: # this user does not yet exist in cache
following_screen_name = api.GetUser(
user_id=following).name
self.cache.append_to_cache(
user_id=following,
screen_name=following_screen_name)
print(
f"Naive BFS: {current_user_screen_name} follows {following_screen_name}. "
f"Requested {following_screen_name}'s screen name from API and cached."
)
else:
following_screen_name = self.cache.read_from_cache(
user_id=following)['screen_name']
print(
f"Naive BFS: {current_user_screen_name} follows {following_screen_name}. "
f"Retrieved {following_screen_name}'s screen name from cache."
)
visited.add(following)
queue.put(following)
current_path = list(path_to[current_user])
current_path.extend([following])
path_to[following] = current_path
if following == target_user:
print(
f"Naive BFS: {following_screen_name} (target user) found! Stopping search."
)
break
except twitter.error.TwitterError as ex:
error_handler.handle(ex, user_id=following)
else:
print('User already visited. Skipping...')
print(num + 1, '/', num_followings)
else:
continue
break
if path_to[target_user] != [start_user] or target_user == start_user:
return path_to[target_user]
else:
return None
def display_paths(self, path):
"""
Prints out path in English
:param path: List of users in path
"""
for idx, user in enumerate(path):
screen_name = self.cache.read_from_cache(
user_id=user)['screen_name']
if idx == 0:
print(f'{screen_name} follows ', end='')
elif idx == len(path) - 1:
print(screen_name)
else:
print(f'{screen_name}, who follows ', end='')
def run_search(self, origin_user, target_user):
"""
Driver code to run search
:param origin_user: User to start search from
:param target_user: User to find path to
:return: List of users in possible path from origin_user to target_user
"""
found_path = self.bfs(start_user=origin_user, target_user=target_user)
self.display_paths(path=found_path)
return found_path
class CoinWallet(object):
'''
Class for a wallet
'''
def __init__(self, wallet_config):
self._errors = []
self._config = {}
self._cache = Cacher({
'accounts': {},
'transactions': {},
'transactiondetails': {},
'balances': {},
'info': {},
})
if type(wallet_config) is dict:
self._config = wallet_config
@property
def provider_id(self):
'''
Property for the provider id
'''
return self.get('id', None)
@property
def enabled(self):
'''
Return the status (enabled/disabled)
'''
return self.get('enabled', False)
def __getitem__(self, key):
'''
Getter for dictionary-line behavior
'''
if key == "currency_symbol":
return self.getCurrencySymbol()
elif key == "currency_code":
return self.getCurrencyCode()
config = getattr(self, '_config')
return config[key]
def __setitem__(self, key, value):
'''
Setter for dictionary-line behavior
'''
config = getattr(self, '_config')
config[key] = value
return setattr(self, '_config', config)
def get(self, key, default=False):
'''
Implementing .get() method for dictionary-line behavior
'''
if self._config.get(key, False):
return self._config.get(key, False)
else:
return default
def haskey(self, key):
'''
Check the existence of key
'''
if key in self._config.keys():
return True
else:
return False
def getNet(self):
'''
Return network value, mainnet or testnet
'''
# check for cached data, use that or get it again
cache_hash = self.getParamHash("provider_id=%s" % self.provider_id)
cached_peerinfo = self._cache.fetch('info', cache_hash)
if cached_peerinfo:
info = cached_peerinfo
else:
info = connector.getInfo(self.provider_id)
self._cache.store('info', cache_hash, info)
is_testnet = False
if info.has_key('testnet'):
is_testnet = info.get('testnet')
if is_testnet is False:
return "mainnet"
elif is_testnet is True:
return "testnet"
else:
# default to mainnet
return "mainnet"
def balance(self):
'''
Get wallet balance
'''
# check for cached data, use that or get it again
cache_hash = self.getParamHash("balance")
cached_object = self._cache.fetch('balance', cache_hash)
if cached_object:
return cached_object.get(self.provider_id, "-")
balance = connector.getBalance(self.provider_id)
# store result in cache
self._cache.store('balance', cache_hash, balance)
return misc.longNumber(balance.get(self.provider_id, "-"))
def getParamHash(self, param=""):
'''
This function takes a string and calculates a sha224 hash out of it.
It is used to hash the input parameters of functions/method in order to
uniquely identify a cached result based only on the input parameters of
the function/method call.
'''
cache_hash = hashlib.sha224(param).hexdigest()
return cache_hash
def listAccounts(self, gethidden=False, getarchived=False):
'''
Get a list of accounts. This method also supports filtering, fetches address for each account etc.
'''
# check for cached data, use that or get it again
cache_hash = self.getParamHash("gethidden=%s&getarchived=%s" % (gethidden, getarchived))
cached_object = self._cache.fetch('accounts', cache_hash)
if cached_object:
return cached_object
# get data from the connector (xxxcoind)
fresh_accounts = connector.listAccounts(gethidden=False, getarchived=False, selected_provider_id=self.provider_id)
# get a list of archived address
address_ignore_list = []
if not getarchived:
ignore_list = accountFilter.objects.filter(status=1)
for ignored_account in ignore_list:
address_ignore_list.append(ignored_account.address.encode('ascii'))
# get a list of hidden accounts
address_hidden_list = []
if not gethidden:
hidden_list = accountFilter.objects.filter(status=2)
for hidden_account in hidden_list:
address_hidden_list.append(hidden_account.address.encode('ascii'))
accountObjects = []
for account_name, account_balance in fresh_accounts.get(self.provider_id, {}).items():
'''
# check all addresses if they are in the archive list
for ignored_address in address_ignore_list:
if ignored_address in account_addresses:
del account_addresses[account_addresses.index(ignored_address)]
# check all addresses if they are in the hidden list
hidden_flag = False
for hidden_address in address_hidden_list:
if hidden_address in account_addresses:
hidden_flag = True
'''
accountObjects.append(CoinAccount({
'name': account_name,
'balance': account_balance,
'currency': self['currency'],
'provider_id': self.provider_id,
'wallet': self,
}))
# cache the result
self._cache.store('accounts', cache_hash, accountObjects)
return accountObjects
def getCurrencySymbol(self):
'''
Return the Unicode currency symbol
'''
return self.get('symbol', None)
def getCurrencyCode(self):
'''
Return the currency code
'''
return self.get('currency', "").lower()
def listTransactions(self, limit=10, start=0):
'''
Return a list of transactions wallet-wide
'''
# check for cached data, use that or get it again
cache_hash = self.getParamHash("limit=%s&start=%s" % (limit, start))
cached_object = self._cache.fetch('transactions', cache_hash)
if cached_object:
return cached_object
transactions = []
transactions_dicts = connector.listTransactionsByAccount("*", self.provider_id, limit, start)
for transaction in transactions_dicts:
transaction['wallet'] = self
transaction['currency'] = self.getCurrencyCode()
transaction['currency_symbol'] = self.getCurrencySymbol()
transaction['provider_id'] = self.provider_id
transactions.append(CoinTransaction(transaction))
self._cache.store('transactions', cache_hash, transactions)
return transactions
def getAccountByName(self, name):
'''
Return CoinAccount() for name
'''
accounts = self.listAccounts(gethidden=True, getarchived=True)
for account in accounts:
if account['name'] == name:
return account
return None
def getTransactionById(self, txid):
'''
Return a transaction by txid
'''
# check for cached data, use that or get it again
cache_hash = self.getParamHash("txid=%s" % (txid))
cached_object = self._cache.fetch('transactiondetails', cache_hash)
if cached_object:
return cached_object
transaction_details = connector.getTransaction(txid, self.provider_id)
transaction_details['currency'] = self.getCurrencyCode()
transaction_details['wallet'] = self
self._cache.store('transactiondetails', cache_hash, transaction_details)
return CoinTransaction(transaction_details)
def getAddressesByAccount(self, account):
'''
Get a list of address for account
'''
addresses_list = connector.getAddressesByAccount(account, self.provider_id)
coinaddresses = []
for address in addresses_list:
coinaddresses.append(CoinAddress(address, account))
return coinaddresses
def getDefaultAccount(self):
'''
Return the CoinAccount object for the default wallet account
'''
accounts = self.listAccounts(gethidden=True, getarchived=True)
for account in accounts:
if len(account['name']) == 0:
return account
else:
return None
def getAccountByAddress(self, address):
'''
Return account by address
'''
accounts = self.listAccounts(gethidden=True, getarchived=True)
target_account = None
for account in accounts:
for account_address in account['addresses']:
if str(address) == str(account_address):
target_account = account
target_account['currency'] = self.getCurrencyCode()
target_account['provider_id'] = self.provider_id
return target_account
else:
return None
def getAccountByIdentifier(self, identifier):
'''
Get account by identifier
'''
list_of_accounts = self.listAccounts(gethidden=True, getarchived=True)
for account in list_of_accounts:
if account.getIdentifier() == identifier:
return account
else:
return None
else:
if event_type == "resourceDown":
if reason is None:
print "Recieved resource down handle, but missing reason."
print __doc__
sys.exit(2)
if share_down is None:
print "Recieved resource down handle, but missing share down."
print __doc__
sys.exit(2)
if date_down is None:
print "Recieved resource down handle, but missing date down."
# We have everything we require to create an event.
# Attempt to find already cached data:
m = MetaDoc(site_name)
c = Cacher("events")
cached_data = c.get_cache()
if cached_data is not None:
processor = Events.from_xml_element(cached_data, Events)
if processor is None:
print "Found previous event cache, but could not load. Please check "
print "\"%s\" for errors. " % c.file_path
print "Halting."
sys.exit(2)
else:
c.remove_cache()
else:
processor = Events()
if event_type == "resourceUp":
e = ResourceUpEntry(date_up, reason, remarks)
else:
def __init__(self, getterType='urlopen', waitTimeBeforeScraping=0):
self.cacher = Cacher()
self.config = Config()
self.getter = Getter(getterType,
waitTimeBeforeScraping=waitTimeBeforeScraping)
class CoinAccount(object):
'''
Class for an account
'''
def __init__(self, accountDetails):
self._errors = []
self._account = {}
self._hidden = False
self._cache = Cacher({
'transactions': {},
'balances': {},
'addressesbyaccount': {},
})
if type(accountDetails) is dict:
self._account = accountDetails
self._provider_id = accountDetails['provider_id']
@property
def provider_id(self):
'''
Property for the provider id
'''
return self.get('provider_id', None)
def __getitem__(self, key):
'''
Getter for dictionary-line behavior
'''
if key == "addresses":
return self.getAddresses()
elif key == "last_activity":
return self.getLastActivity()
elif key == "currency_symbol":
return self.getCurrencySymbol()
elif key == "currency_code":
return self.getCurrencyCode()
elif key == 'identifier':
return self.getIdentifier()
account = getattr(self, '_account')
return account.get(key, None)
def __setitem__(self, key, value):
'''
Setter for dictionary-line behavior
'''
account = getattr(self, '_account')
account[key] = value
return setattr(self, '_account', account)
def get(self, key, default=False):
'''
Getter for dictionary-line behavior
'''
if self._account.get(key, False):
return self._account.get(key, False)
else:
return default
def haskey(self, key):
'''
Check the existence of key
'''
if key in self._account.keys():
return True
else:
return False
def getParamHash(self, param=""):
'''
This function takes a string and calculates a sha224 hash out of it.
It is used to hash the input parameters of functions/method in order to
uniquely identify a cached result based only on the input parameters of
the function/method call.
'''
cache_hash = hashlib.sha224(param).hexdigest()
return cache_hash
def getIdentifier(self):
'''
There is no unique identifier for an account in a xxxcoind daemon
so lets make one. Hopefully the below hashing method will uniquely
identify an account for us
'''
unique_string = "provider_id=%s&name=%s¤cy=%s" % (self['provider_id'], self['name'], self['currency'])
identifier = hashlib.sha1(unique_string).hexdigest()
return identifier
def isDefault(self):
'''
Return bool whether this is a default account or not
'''
if self._account['name'] == u"":
self._hidden = True
return True
else:
return False
def getBalance(self):
'''
Return the account balance
'''
balance = connector.getBalance(self.provider_id, self['name'])
return misc.longNumber(balance)
def isHidden(self):
'''
Return bool if this account is hidden
'''
return self._hidden or self._account['hidden'] or self.isDefault()
def getAddresses(self):
'''
Get the address for an account name
'''
# check for cached data, use that or get it again
cache_hash = self.getParamHash("name=%s" % (self['name']))
cached_object = self._cache.fetch('addressesbyaccount', cache_hash)
if cached_object:
return cached_object
addresses = connector.getAddressesByAccount(self['name'], self.provider_id)
addresses_list = []
for address in addresses:
coinaddr = CoinAddress(address, self)
addresses_list.append(coinaddr)
# cache the result
self._cache.store('addressesbyaccount', cache_hash, addresses_list)
return addresses_list
def getAddressesCount(self):
'''
Return the number of address under this account
'''
addresses = self.getAddresses()
return len(addresses)
def getLastActivity(self):
'''
Return the date of the last activity
'''
last_transaction = self.listTransactions(1, 0)
if last_transaction:
last_activity = misc.twitterizeDate(last_transaction[0]['time'])
else:
last_activity = "never"
self['last_activity'] = last_activity
return last_activity
def getCurrencySymbol(self):
'''
Return the Unicode currency symbol
'''
return misc.getCurrencySymbol(connector, self.getCurrencyCode())
def getCurrencyCode(self):
'''
Return the currency code
'''
return self.get('currency', "").lower()
def listTransactions(self, limit=100000, start=0, orderby='time', reverse=True):
'''
Get a list of transactions by account name and provider_id
'''
cache_hash = self.getParamHash("limit=%s&start=%sorderby=%s&reverse=%s" % (limit, start, orderby, reverse))
cached_object = self._cache.fetch('transactions', cache_hash)
if cached_object:
return cached_object
transactions = []
transaction_list = connector.listTransactionsByAccount(self['name'], self['provider_id'], limit, start)
for entry in transaction_list:
if entry.get('address', False):
entry['address'] = CoinAddress(entry['address'], self)
# give out a provider id and a currency code to the transaction dict
entry['provider_id'] = self.provider_id
entry['currency'] = self['currency']
if entry['category'] == 'receive':
entry['source_address'] = CoinAddress(entry.get('details', {}).get('sender_address', False), "This is a sender address!")
elif entry['category'] == 'send':
entry['source_addresses'] = self['wallet'].getAddressesByAccount(entry['account'])
entry['wallet'] = self['wallet']
coin_transaction = CoinTransaction(entry)
transactions.append(coin_transaction)
# sort result
transactions = sorted(transactions, key=lambda transaction: transaction[orderby], reverse=reverse)
# cache the result
self._cache.store('transactions', cache_hash, transactions)
return transactions
def process_message(data, slack_client):
global bot_message_count
print "process message", data
data = Data(**data)
global slack_users
slack_users = SlackUsers.generate(slack_client)
try:
if slack_users.hes_zeebz == data.user:
Cacher.add_message_by_user(slack_users.hes_zeebz, data.text)
except Exception:
pass
get_or_set_bot_messages(data, slack_client)
print "BOT MESSAGE COUNT", bot_message_count
# We don't care what Phteven Marquarst says
if data.user == slack_users.phteven:
print "its Phteven..."
return
rules = PhtevenRules(data)
rules.add_many_exclusive([
# If @Phteven gets mentioned
(
lambda data: slack_users.phteven.at in data.text,
lambda: "^ %s" % slack_users.not_stephanie.at
),
# Copy smarx
(
lambda data: random.random() < 0.05 and data.user == slack_users.hes_zeebz,
lambda: data.text
),
# Smarx has an elipsis
(
lambda data: u"\u2026" in data.text and data.user == slack_users.hes_zeebz,
lambda: "Nice try, Steven J Smarx"
),
# Are we in burgertory?
(
lambda data: 'burgertory' in data.text.lower() or 'burgatory' in data.text.lower(),
lambda: "We're always in burgertory"
),
# Zeebs trying to be funny
(
lambda data: PhtevenRegex.thats_something.search(data.text) and data.user == slack_users.whos_zeebz,
lambda: "Shut up, %s" % slack_users.whos_zeebz.at
),
# Am i saying meow?
(
lambda data: PhtevenRegex.meow_now.search(data.text),
lambda: "Right meow?"
),
# That's <blank>
(
lambda data: PhtevenRegex.thats_something.search(data.text),
lambda: "You're %s" % PhtevenRegex.thats_something.search(data.text).group(2)
),
# You're <blank>
(
lambda data: PhtevenRegex.youre_something.search(data.text),
lambda: "No, you're %s!!" % PhtevenRegex.youre_something.search(data.text).group(2)
),
# Childish. References to poop things
(
lambda data: any(word in ['poop', 'turd'] for word in data.text.split(' ')),
lambda: ":poop:"
),
# if someone talks about Seattle
(
lambda data: "seattle" in data.text.lower(),
lambda: "Seattle?? Isn't that where %s lives?" % slack_users.whos_zeebz.at
),
# Vegas??
(
lambda data: "vegas" in data.text.lower(),
lambda: "%s lives in Vegas!" % slack_users.hes_zeebz.at
),
# If someone just says hello
(
lambda data: 'hello' in data.text.lower() and data.user == 'U12D20DCP',
lambda: "Traduccion: Hola."
),
# Did someone say snackatizer??
(
lambda data: "snack" in data.text,
lambda: "Mayhaps mean snackatizer???"
),
# Myth busted
(
lambda data: PhtevenRegex.myth_busted.search(data.text),
lambda: random.choice(PhtevenPhrases.myth_busted_phrases)
),
# What dis is?
(
lambda data: PhtevenRegex.whatdisis.search(data.text),
lambda: "YOU KNOW WHAT DIS IS"
),
# If something isn't real
(
lambda data: PhtevenRegex.arentreal.search(data.text),
lambda: "You're not real."
),
])
rules.add_many_inclusive([
(
check_bot_message_count,
lambda: "TRIPLE BOT MESSAGE COMBO!!!"
),
# User id?
(
who_is,
lambda: sendback_users(data)
),
# Science questions
(
lambda data: random.random() < 0.03,
say_something_sciencey
),
# room, oasted
(
room_oasted,
lambda: say_room_oasted(data)
),
# That's what she said
(
lambda data: "harder" in data.text,
lambda: "That's what she said"
),
# Say what smarx said...but a while ago
(
test_smarx,
say_smarx_and_reset
),
# Say what stephen said...but a while ago
(
test_stephen,
say_stephen_and_reset
),
# Party pooper
(
lambda data: 'party' in data.text.lower(),
lambda: "More like party pooper. lawl"
),
# Smarx was president of the juggling club
(
lambda data: 'juggl' in data.text.lower(),
lambda: "Did you know %s was president of his juggling club in school?" % slack_users.hes_zeebz.at
),
# WAHH
(
lambda data: random.random() < 0.02,
lambda: random.choice(PhtevenPhrases.phteven_phrases)
),
# As it were
(
lambda data: 'as it were' in data.text.lower(),
lambda: "Pindeed"
),
# I hardly knew her
(
lambda data: PhtevenRegex.baylor.search(data.text),
lambda: "%s, I hardly know her" % PhtevenRegex.baylor.search(data.text).group(1)
),
# Be quiet, zeebz
(
lambda data: (data.user == slack_users.whos_zeebz) and random.random() < 0.01,
lambda: "Shut up, %s" % slack_users.whos_zeebz.at
),
# Foldable bluetooth keyboard
(
lambda data: 'foldable bluetooth keyboard' in data.text.lower(),
lambda: "You're a foldable bluetooth keyboard"
),
# Giphy
(
lambda data: data.text.startswith('/giphy') and random.random() < 0.50,
lambda: random.choice(PhtevenPhrases.giphy_phrases)
),
# Hello
(
lambda data: 'hello' in data.text.lower(),
lambda: "Oh. Hello."
),
])
processor = PhtevenProcessor(rules)
outputs.extend(processor.process())
print("文章来源:" + D["SOURCE"])
print("采集时间:" + D["PickingTime"])
print("正文内容:" + D["CONTENT"].decode("utf-8"))
return D
def showMenu():
print(
"[1]文章点赞（Vote）\t[2]文章分享（Share）\t[3]文章保存（Save）\n\
[4]文章评论（Remark）\t[5]邮件反馈（Email）\t[ENTER]下一篇"
)
cmd = input("请输入指令(Q退出):")
return cmd
cc = Cacher()
cc.start()
input("\t欢迎使用简致阅读\n\t按任意件开启阅读")
D = showArticle()
cmd = showMenu()
while True:
if cmd == "Q":
break
elif cmd == "1":
pass
elif cmd == "2":
pass
elif cmd == "3":
pass
elif cmd == "4":
def test_stephen(_):
return len(Cacher.get_messages_by_user(slack_users.not_stephanie)) >= 25
def say_stephen_and_reset():
to_say = Cacher.get_message_by_user(slack_users.not_stephanie, randomize=True)
Cacher.reset_messages_by_user(slack_users.not_stephanie, randomize=True)
return to_say
class Extractor(object):
def __init__(self):
self._cache_enabled = False
self._cache_connection_port = None
self._cache_connection_host = None
self._cacher = None
self._arg = {
"country_codes": ["hrv", "usa"],
"indicator_codes": ["SP.POP.TOTL"],
"start_date": 1980,
"end_date": 2010,
"pause": 0
}
def arg(self):
return copy.deepcopy(self._arg)
def normalize(self, arg):
if arg == None:
arg = self.arg()
if "interval" in arg:
arg["start_date"], arg["end_date"] = arg["interval"]
capitalize_list = lambda elems: [el.upper() for el in elems]
arg["country_codes"] = capitalize_list(arg["country_codes"])
return arg
def grab(self, arg=None):
rca_indicators = []
wb_indicators = []
try:
import local_storage.rca.api
# Check if indicator is in RCA dataset, if not then it must be in WB.
# TODO: Make indicator type checking universal.
available_rca_indicators = set(
local_storage.rca.api.all_indicators())
for indicator_code in arg["indicator_codes"]:
if indicator_code in available_rca_indicators:
rca_indicators.append(indicator_code)
else:
wb_indicators.append(indicator_code)
# Grab RCA data.
rca_arg = copy.deepcopy(arg)
rca_arg["indicator_codes"] = rca_indicators
rca_countries = self._grab_from_api(rca_arg, local_storage.rca.api)
except ImportError:
print(
"Can't import local_storage.rca.api! Will assume there aren't any indicators from there"
)
rca_countries = []
wb_indicators = arg["indicator_codes"]
# Grab WB data.
wb_arg = copy.deepcopy(arg)
wb_arg["indicator_codes"] = wb_indicators
wb_countries = self._grab_from_api(wb_arg, wb.api)
#TODO: Separate function for merging two or more list of countries.
# Not assuming that wb and rca countries are in corresponding order in lists.
all_codes = set()
wb_dict = {}
for wb_country in wb_countries:
all_codes.add(wb_country.code)
wb_dict[wb_country.code] = wb_country
rca_dict = {}
for rca_country in rca_countries:
all_codes.add(rca_country.code)
rca_dict[rca_country.code] = rca_country
combined_countries = []
for code in all_codes:
current_countries = []
try:
current_countries.append(wb_dict[code])
except KeyError:
pass
try:
current_countries.append(rca_dict[code])
except KeyError:
pass
if len(current_countries) != 0:
if len(current_countries) > 1:
current_countries[0].merge_with_country(
current_countries[1])
combined_countries.append(current_countries[0])
return combined_countries
# #TODO: Merge RCA and WB data.
# # Assumptions is that rca_countries and wb_countries have elements in same order (maybe write unit test for that).
# for wb_country,rca_country in zip(wb_countries,rca_countries):
# wb_country.merge_with_country(rca_country)
#
# return wb_countries
def _grab_from_api(self, arg=None, api=wb.api):
#TODO: think about using kwargs
"""
possible args are:
1) the default by leaving None, defined in arg()
2) setting your own such as:
{"country_codes" : ["hrv", "usa", "chn"],
"indicator_codes" : ["SP.POP.TOTL"],
"start_date":1980,
"end_date":2010}
3) using interval to set the dates more quickly (overrides start/end_date)
{"country_codes" : ["hrv", "usa", "chn"],
"indicator_codes" : ["SP.POP.TOTL"],
"interval":(1980,2010)}
"""
arg = self.normalize(arg)
if self._cache_enabled:
countries = self._cacher.retreive(arg)
if countries: self.cache_hit = True
else: self.cache_hit = False
if not self._cache_enabled or not self.cache_hit:
countries = api.query_multiple_data(arg["country_codes"],
arg["indicator_codes"],
arg["start_date"],
arg["end_date"])
if arg["pause"]:
time.sleep(arg["pause"])
if self._cache_enabled and not self.cache_hit:
self._cache(countries)
return countries
def grab_metadata(self, what="countries", api=wb.api):
"""
@param what: string of what to get
- 'countries' - list of countries
- 'indicators' - list of indicators
"""
if what == "countries":
return api.all_countries()
elif what == "indicators":
return api.all_indicators()
def enable_cache(self, host="localhost", port=27017, test=False):
"""
@param host: the hostname of the server where mongodb is running
@param port: port mongodb is listening on
@param test: True if you want to work on a db separate than the main cache
"""
self._cache_connection_host = host
self._cache_connection_port = port
self._cache_enabled = True
self._cacher = Cacher(self._cache_connection_host,
self._cache_connection_port, test)
def disable_cache(self):
self._cache_enabled = False
def clear_cache(self):
"""
clear everything from the cache
"""
self._cacher.clear()
def _cache(self, countries):
if self._cache_enabled:
self._cacher.cache(countries)
else:
return
def is_cached(self, arg):
arg = self.normalize(arg)
countries = self._cacher.retreive(arg)
if countries: return True
else: return False
def was_cached(self):
"""
tells if the data was found in the cache on the last grab
"""
return self.cache_hit
class CoinAccount(object):
'''
Class for an account
'''
def __init__(self, accountDetails):
self._errors = []
self._account = {}
self._hidden = False
self._cache = Cacher({
'transactions': {},
'balances': {},
'addressesbyaccount': {},
})
if type(accountDetails) is dict:
self._account = accountDetails
self._provider_id = accountDetails['provider_id']
@property
def provider_id(self):
'''
Property for the provider id
'''
return self.get('provider_id', None)
def __getitem__(self, key):
'''
Getter for dictionary-line behavior
'''
if key == "addresses":
return self.getAddresses()
elif key == "last_activity":
return self.getLastActivity()
elif key == "currency_symbol":
return self.getCurrencySymbol()
elif key == "currency_code":
return self.getCurrencyCode()
elif key == 'identifier':
return self.getIdentifier()
account = getattr(self, '_account')
return account.get(key, None)
def __setitem__(self, key, value):
'''
Setter for dictionary-line behavior
'''
account = getattr(self, '_account')
account[key] = value
return setattr(self, '_account', account)
def get(self, key, default=False):
'''
Getter for dictionary-line behavior
'''
if self._account.get(key, False):
return self._account.get(key, False)
else:
return default
def haskey(self, key):
'''
Check the existence of key
'''
if key in self._account.keys():
return True
else:
return False
def getParamHash(self, param=""):
'''
This function takes a string and calculates a sha224 hash out of it.
It is used to hash the input parameters of functions/method in order to
uniquely identify a cached result based only on the input parameters of
the function/method call.
'''
cache_hash = hashlib.sha224(param).hexdigest()
return cache_hash
def getIdentifier(self):
'''
There is no unique identifier for an account in a xxxcoind daemon
so lets make one. Hopefully the below hashing method will uniquely
identify an account for us
'''
unique_string = "provider_id=%s&name=%s¤cy=%s" % (
self['provider_id'], self['name'], self['currency'])
identifier = hashlib.sha1(unique_string).hexdigest()
return identifier
def isDefault(self):
'''
Return bool whether this is a default account or not
'''
if self._account['name'] == u"":
self._hidden = True
return True
else:
return False
def getBalance(self):
'''
Return the account balance
'''
balance = connector.getBalance(self.provider_id, self['name'])
return misc.longNumber(balance)
def isHidden(self):
'''
Return bool if this account is hidden
'''
return self._hidden or self._account['hidden'] or self.isDefault()
def getAddresses(self):
'''
Get the address for an account name
'''
# check for cached data, use that or get it again
cache_hash = self.getParamHash("name=%s" % (self['name']))
cached_object = self._cache.fetch('addressesbyaccount', cache_hash)
if cached_object:
return cached_object
addresses = connector.getAddressesByAccount(self['name'],
self.provider_id)
addresses_list = []
for address in addresses:
coinaddr = CoinAddress(address, self)
addresses_list.append(coinaddr)
# cache the result
self._cache.store('addressesbyaccount', cache_hash, addresses_list)
return addresses_list
def getAddressesCount(self):
'''
Return the number of address under this account
'''
addresses = self.getAddresses()
return len(addresses)
def getLastActivity(self):
'''
Return the date of the last activity
'''
last_transaction = self.listTransactions(1, 0)
if last_transaction:
last_activity = misc.twitterizeDate(last_transaction[0]['time'])
else:
last_activity = "never"
self['last_activity'] = last_activity
return last_activity
def getCurrencySymbol(self):
'''
Return the Unicode currency symbol
'''
return misc.getCurrencySymbol(connector, self.getCurrencyCode())
def getCurrencyCode(self):
'''
Return the currency code
'''
return self.get('currency', "").lower()
def listTransactions(self,
limit=100000,
start=0,
orderby='time',
reverse=True):
'''
Get a list of transactions by account name and provider_id
'''
cache_hash = self.getParamHash(
"limit=%s&start=%sorderby=%s&reverse=%s" %
(limit, start, orderby, reverse))
cached_object = self._cache.fetch('transactions', cache_hash)
if cached_object:
return cached_object
transactions = []
transaction_list = connector.listTransactionsByAccount(
self['name'], self['provider_id'], limit, start)
for entry in transaction_list:
if entry.get('address', False):
entry['address'] = CoinAddress(entry['address'], self)
# give out a provider id and a currency code to the transaction dict
entry['provider_id'] = self.provider_id
entry['currency'] = self['currency']
if entry['category'] == 'receive':
entry['source_address'] = CoinAddress(
entry.get('details', {}).get('sender_address', False),
"This is a sender address!")
elif entry['category'] == 'send':
entry['source_addresses'] = self[
'wallet'].getAddressesByAccount(entry['account'])
entry['wallet'] = self['wallet']
coin_transaction = CoinTransaction(entry)
transactions.append(coin_transaction)
# sort result
transactions = sorted(transactions,
key=lambda transaction: transaction[orderby],
reverse=reverse)
# cache the result
self._cache.store('transactions', cache_hash, transactions)
return transactions
class ErrorHandler:
def __init__(self, api):
self.api = api
self.cache = Cacher()
def api_test(self):
"""
This instructs user to rerun program in a few minutes if rate limit has exceeded.
If rate limit has exceeded before running program, sleep_on_rate_limit won't work properly.
"""
try:
_ = self.api.GetUser(user_id=813286)
except twitter.error.TwitterError as ex:
if ex.message[0]['code'] == 88:
sys.exit(
'Rate limit for API calls has exceeded. Rerun program in a few minutes.'
)
else:
raise ex
def handle_50(self, arg_dict):
"""
Error 50 means that the user is not found.
:param arg_dict: All the arguments the error handling will need. Here, we only need user_id.
"""
self.cache.remove_all_occurrences(user_id=arg_dict['user_id'])
print(
f"Requested user not found. Removed all occurrences of {arg_dict['user_id']} in cache"
)
def handle_63(self, arg_dict):
"""
Error 63 means that the requested user has been suspended. We simply skip the user.
:param arg_dict: All the arguments the error handling will need. Here, we only need user_id.
"""
self.cache.remove_all_occurrences(user_id=arg_dict['user_id'])
print(
f"Requested user suspended. Removed all occurrences of {arg_dict['user_id']} in cache"
)
def handle_88(self):
"""
Error 88 means that the Twitter API rate limit has been exceeded. We keep attempting a connection until it
resolves. This bit of code should not be executed, as we specified sleep_on_rate_limit in the API constructor
and tested the connection before hand with self.api_test().
"""
while True:
try:
_ = self.api.GetUser(user_id=813286)
except twitter.error.TwitterError as ex:
if ex.message[0]['code'] == 88:
print('Waiting for rate limit to reset')
sleep(60)
continue
else:
raise ex
else:
break
def handle(self, ex, **kwargs):
"""
This acts as a switchboard for each handling method. Note how all the error handlers' names follow the style
self.handle_{code}().
:param ex: The raised exception. We need it to identify the error code raised.
:param kwargs: The specific arguments we need in order to fix the error.
"""
code = ex.message[0]['code']
try:
_ = getattr(self, 'handle_' + str(code))(kwargs)
except AttributeError:
print('No handler has been written for this error. Whoops.')
raise ex
class Extractor(object):
def __init__(self):
self._cache_enabled = False
self._cache_connection_port = None
self._cache_connection_host = None
self._cacher = None
self._arg = {"country_codes" : ["hrv", "usa"],
"indicator_codes" : ["SP.POP.TOTL"],
"start_date" : 1980,
"end_date" : 2010,
"pause" : 0}
def arg(self):
return copy.deepcopy(self._arg)
def normalize(self, arg):
if arg==None:
arg = self.arg()
if "interval" in arg:
arg["start_date"], arg["end_date"] = arg["interval"]
capitalize_list = lambda elems : [el.upper() for el in elems]
arg["country_codes"] = capitalize_list(arg["country_codes"])
return arg
def grab(self, arg=None):
rca_indicators = []
wb_indicators = []
try:
import local_storage.rca.api
# Check if indicator is in RCA dataset, if not then it must be in WB.
# TODO: Make indicator type checking universal.
available_rca_indicators = set(local_storage.rca.api.all_indicators())
for indicator_code in arg["indicator_codes"]:
if indicator_code in available_rca_indicators:
rca_indicators.append(indicator_code)
else:
wb_indicators.append(indicator_code)
# Grab RCA data.
rca_arg = copy.deepcopy(arg)
rca_arg["indicator_codes"] = rca_indicators
rca_countries = self._grab_from_api(rca_arg, local_storage.rca.api)
except ImportError:
print("Can't import local_storage.rca.api! Will assume there aren't any indicators from there")
rca_countries = []
wb_indicators = arg["indicator_codes"]
# Grab WB data.
wb_arg = copy.deepcopy(arg)
wb_arg["indicator_codes"] = wb_indicators
wb_countries = self._grab_from_api(wb_arg, wb.api)
#TODO: Separate function for merging two or more list of countries.
# Not assuming that wb and rca countries are in corresponding order in lists.
all_codes = set()
wb_dict = {}
for wb_country in wb_countries:
all_codes.add(wb_country.code)
wb_dict[wb_country.code] = wb_country
rca_dict = {}
for rca_country in rca_countries:
all_codes.add(rca_country.code)
rca_dict[rca_country.code] = rca_country
combined_countries = []
for code in all_codes:
current_countries = []
try: current_countries.append(wb_dict[code])
except KeyError: pass
try: current_countries.append(rca_dict[code])
except KeyError: pass
if len(current_countries)!=0:
if len(current_countries)>1:
current_countries[0].merge_with_country(current_countries[1])
combined_countries.append(current_countries[0])
return combined_countries
# #TODO: Merge RCA and WB data.
# # Assumptions is that rca_countries and wb_countries have elements in same order (maybe write unit test for that).
# for wb_country,rca_country in zip(wb_countries,rca_countries):
# wb_country.merge_with_country(rca_country)
#
# return wb_countries
def _grab_from_api(self, arg=None, api=wb.api):
#TODO: think about using kwargs
"""
possible args are:
1) the default by leaving None, defined in arg()
2) setting your own such as:
{"country_codes" : ["hrv", "usa", "chn"],
"indicator_codes" : ["SP.POP.TOTL"],
"start_date":1980,
"end_date":2010}
3) using interval to set the dates more quickly (overrides start/end_date)
{"country_codes" : ["hrv", "usa", "chn"],
"indicator_codes" : ["SP.POP.TOTL"],
"interval":(1980,2010)}
"""
arg = self.normalize(arg)
if self._cache_enabled:
countries = self._cacher.retreive(arg)
if countries: self.cache_hit = True
else: self.cache_hit = False
if not self._cache_enabled or not self.cache_hit:
countries = api.query_multiple_data(
arg["country_codes"], arg["indicator_codes"],
arg["start_date"], arg["end_date"])
if arg["pause"]:
time.sleep(arg["pause"])
if self._cache_enabled and not self.cache_hit:
self._cache(countries)
return countries
def grab_metadata(self, what="countries", api = wb.api):
"""
@param what: string of what to get
- 'countries' - list of countries
- 'indicators' - list of indicators
"""
if what=="countries":
return api.all_countries()
elif what=="indicators":
return api.all_indicators()
def enable_cache(self, host="localhost", port=27017, test=False):
"""
@param host: the hostname of the server where mongodb is running
@param port: port mongodb is listening on
@param test: True if you want to work on a db separate than the main cache
"""
self._cache_connection_host = host
self._cache_connection_port = port
self._cache_enabled = True
self._cacher = Cacher(self._cache_connection_host, self._cache_connection_port, test)
def disable_cache(self):
self._cache_enabled = False
def clear_cache(self):
"""
clear everything from the cache
"""
self._cacher.clear()
def _cache(self,countries):
if self._cache_enabled:
self._cacher.cache(countries)
else: return
def is_cached(self, arg):
arg = self.normalize(arg)
countries = self._cacher.retreive(arg)
if countries: return True
else: return False
def was_cached(self):
"""
tells if the data was found in the cache on the last grab
"""
return self.cache_hit
def __init__(self, api):
self.api = api
self.cache = Cacher()
class CoinWallet(object):
'''
Class for a wallet
'''
def __init__(self, wallet_config):
self._errors = []
self._config = {}
self._cache = Cacher({
'accounts': {},
'transactions': {},
'transactiondetails': {},
'balances': {},
'info': {},
})
if type(wallet_config) is dict:
self._config = wallet_config
@property
def provider_id(self):
'''
Property for the provider id
'''
return self.get('id', None)
@property
def enabled(self):
'''
Return the status (enabled/disabled)
'''
return self.get('enabled', False)
def __getitem__(self, key):
'''
Getter for dictionary-line behavior
'''
if key == "currency_symbol":
return self.getCurrencySymbol()
elif key == "currency_code":
return self.getCurrencyCode()
config = getattr(self, '_config')
return config[key]
def __setitem__(self, key, value):
'''
Setter for dictionary-line behavior
'''
config = getattr(self, '_config')
config[key] = value
return setattr(self, '_config', config)
def get(self, key, default=False):
'''
Implementing .get() method for dictionary-line behavior
'''
if self._config.get(key, False):
return self._config.get(key, False)
else:
return default
def haskey(self, key):
'''
Check the existence of key
'''
if key in self._config.keys():
return True
else:
return False
def getNet(self):
'''
Return network value, mainnet or testnet
'''
# check for cached data, use that or get it again
cache_hash = self.getParamHash("provider_id=%s" % self.provider_id)
cached_peerinfo = self._cache.fetch('info', cache_hash)
if cached_peerinfo:
info = cached_peerinfo
else:
info = connector.getInfo(self.provider_id)
self._cache.store('info', cache_hash, info)
is_testnet = False
if info.has_key('testnet'):
is_testnet = info.get('testnet')
if is_testnet is False:
return "mainnet"
elif is_testnet is True:
return "testnet"
else:
# default to mainnet
return "mainnet"
def balance(self):
'''
Get wallet balance
'''
# check for cached data, use that or get it again
cache_hash = self.getParamHash("balance")
cached_object = self._cache.fetch('balance', cache_hash)
if cached_object:
return cached_object.get(self.provider_id, "-")
balance = connector.getBalance(self.provider_id)
# store result in cache
self._cache.store('balance', cache_hash, balance)
return misc.longNumber(balance.get(self.provider_id, "-"))
def getParamHash(self, param=""):
'''
This function takes a string and calculates a sha224 hash out of it.
It is used to hash the input parameters of functions/method in order to
uniquely identify a cached result based only on the input parameters of
the function/method call.
'''
cache_hash = hashlib.sha224(param).hexdigest()
return cache_hash
def listAccounts(self, gethidden=False, getarchived=False):
'''
Get a list of accounts. This method also supports filtering, fetches address for each account etc.
'''
# check for cached data, use that or get it again
cache_hash = self.getParamHash("gethidden=%s&getarchived=%s" %
(gethidden, getarchived))
cached_object = self._cache.fetch('accounts', cache_hash)
if cached_object:
return cached_object
# get data from the connector (xxxcoind)
fresh_accounts = connector.listAccounts(
gethidden=False,
getarchived=False,
selected_provider_id=self.provider_id)
# get a list of archived address
address_ignore_list = []
if not getarchived:
ignore_list = accountFilter.objects.filter(status=1)
for ignored_account in ignore_list:
address_ignore_list.append(
ignored_account.address.encode('ascii'))
# get a list of hidden accounts
address_hidden_list = []
if not gethidden:
hidden_list = accountFilter.objects.filter(status=2)
for hidden_account in hidden_list:
address_hidden_list.append(
hidden_account.address.encode('ascii'))
accountObjects = []
for account_name, account_balance in fresh_accounts.get(
self.provider_id, {}).items():
'''
# check all addresses if they are in the archive list
for ignored_address in address_ignore_list:
if ignored_address in account_addresses:
del account_addresses[account_addresses.index(ignored_address)]
# check all addresses if they are in the hidden list
hidden_flag = False
for hidden_address in address_hidden_list:
if hidden_address in account_addresses:
hidden_flag = True
'''
accountObjects.append(
CoinAccount({
'name': account_name,
'balance': account_balance,
'currency': self['currency'],
'provider_id': self.provider_id,
'wallet': self,
}))
# cache the result
self._cache.store('accounts', cache_hash, accountObjects)
return accountObjects
def getCurrencySymbol(self):
'''
Return the Unicode currency symbol
'''
return self.get('symbol', None)
def getCurrencyCode(self):
'''
Return the currency code
'''
return self.get('currency', "").lower()
def listTransactions(self, limit=10, start=0):
'''
Return a list of transactions wallet-wide
'''
# check for cached data, use that or get it again
cache_hash = self.getParamHash("limit=%s&start=%s" % (limit, start))
cached_object = self._cache.fetch('transactions', cache_hash)
if cached_object:
return cached_object
transactions = []
transactions_dicts = connector.listTransactionsByAccount(
"*", self.provider_id, limit, start)
for transaction in transactions_dicts:
transaction['wallet'] = self
transaction['currency'] = self.getCurrencyCode()
transaction['currency_symbol'] = self.getCurrencySymbol()
transaction['provider_id'] = self.provider_id
transactions.append(CoinTransaction(transaction))
self._cache.store('transactions', cache_hash, transactions)
return transactions
def getAccountByName(self, name):
'''
Return CoinAccount() for name
'''
accounts = self.listAccounts(gethidden=True, getarchived=True)
for account in accounts:
if account['name'] == name:
return account
return None
def getTransactionById(self, txid):
'''
Return a transaction by txid
'''
# check for cached data, use that or get it again
cache_hash = self.getParamHash("txid=%s" % (txid))
cached_object = self._cache.fetch('transactiondetails', cache_hash)
if cached_object:
return cached_object
transaction_details = connector.getTransaction(txid, self.provider_id)
transaction_details['currency'] = self.getCurrencyCode()
transaction_details['wallet'] = self
self._cache.store('transactiondetails', cache_hash,
transaction_details)
return CoinTransaction(transaction_details)
def getAddressesByAccount(self, account):
'''
Get a list of address for account
'''
addresses_list = connector.getAddressesByAccount(
account, self.provider_id)
coinaddresses = []
for address in addresses_list:
coinaddresses.append(CoinAddress(address, account))
return coinaddresses
def getDefaultAccount(self):
'''
Return the CoinAccount object for the default wallet account
'''
accounts = self.listAccounts(gethidden=True, getarchived=True)
for account in accounts:
if len(account['name']) == 0:
return account
else:
return None
def getAccountByAddress(self, address):
'''
Return account by address
'''
accounts = self.listAccounts(gethidden=True, getarchived=True)
target_account = None
for account in accounts:
for account_address in account['addresses']:
if str(address) == str(account_address):
target_account = account
target_account['currency'] = self.getCurrencyCode()
target_account['provider_id'] = self.provider_id
return target_account
else:
return None
def getAccountByIdentifier(self, identifier):
'''
Get account by identifier
'''
list_of_accounts = self.listAccounts(gethidden=True, getarchived=True)
for account in list_of_accounts:
if account.getIdentifier() == identifier:
return account
else:
return None
def __init__(self, api, search_radius=1000, auto_next=False):
self.api = api
self.search_radius = search_radius
self.auto_next = auto_next
self.cache = Cacher('cache2.txt')
#
# Read and analyze output from ptrace-sampler
#
import sys
import os
import time
import subprocess
import select
import re
import resource
from lib_finder import LibFinder
from cacher import Cacher
cache = Cacher()
class Mappings:
def __init__(self):
self.origLines = []
self.mappings = []
def parseLine(self, line):
self.origLines.append(line)
# b74b3000-b760f000 r-xp 00000000 09:00 9593032 /lib/tls/i686/cmov/libc-2.9.so
#(addrs, perms, offset, dev, inode, path) = line.split()
parts = line.split()
addrs = parts[0].split('-')
class BidirectionalBFS:
def __init__(self, api, search_radius=1000, auto_next=False):
self.api = api
self.search_radius = search_radius
self.auto_next = auto_next
self.cache = Cacher('cache2.txt')
def origin_following_bfs(self, start_user):
"""
If the origin follows user A and B, and user B follows C, a graph can be drown as such:
Origin ---> A
---> B ----> C
Every yield expands the radius of the search by 1, until it hits search_radius
:param start_user: The user from which to start the search
:yield: All paths possible up to the current radius, in format set({user_id: path_to_that_user_id}, etc.)
"""
api = self.api
error_handler = ErrorHandler(api=api)
queue = Queue()
queue.put(start_user)
visited = {start_user}
path_to = defaultdict(lambda: [start_user])
_ = path_to[start_user]
for current_radius in range(self.search_radius):
if queue.qsize() == 0:
print('Queue empty. Ending origin BFS.')
return
current_user = queue.get()
if self.cache.read_from_cache(
user_id=current_user
) is None: # current_user does not yet exist in cache
current_user_followings = api.GetFriendIDs(
user_id=current_user)
current_user_screen_name = api.GetUser(
user_id=current_user).name
self.cache.append_to_cache(
user_id=current_user,
screen_name=current_user_screen_name,
following_ids=current_user_followings)
print(
f"Origin BFS: requested {current_user_screen_name}'s screen name and followings from API. "
f"Commencing search.")
else:
cached_data = self.cache.read_from_cache(user_id=current_user)
current_user_screen_name = cached_data[
'screen_name'] # if user_id exists as a key in cache,
# screen_name will always exist as well
if not cached_data[
'following']: # i.e. following list does not yet exist in cache
current_user_followings = api.GetFriendIDs(
user_id=current_user)
self.cache.append_to_cache(
user_id=current_user,
screen_name=current_user_screen_name,
following_ids=current_user_followings)
print(
f"Origin BFS: retrieved {current_user_screen_name}'s screen name from cache. "
f"Requested followings from API. Commencing search.")
else:
current_user_followings = cached_data['following']
print(
f"Origin BFS: retrieved {current_user_screen_name}'s screen name AND followings from cache. "
f"Today is a good day. Commencing search.")
num_followings = len(current_user_followings)
for num, following in enumerate(current_user_followings):
if following not in visited:
try:
if self.cache.read_from_cache(
user_id=following
) is None: # this user does not yet exist in cache
following_screen_name = api.GetUser(
user_id=following).name
self.cache.append_to_cache(
user_id=following,
screen_name=following_screen_name)
print(
f"Origin BFS: {current_user_screen_name} follows {following_screen_name}. "
f"Requested {following_screen_name}'s screen name from API and cached."
)
else:
following_screen_name = self.cache.read_from_cache(
user_id=following)['screen_name']
print(
f"Origin BFS: {current_user_screen_name} follows {following_screen_name}. "
f"Retrieved {following_screen_name}'s screen name from cache."
)
visited.add(following)
queue.put(following)
current_path = list(path_to[current_user])
current_path.extend([following])
path_to[following] = current_path
except twitter.error.TwitterError as ex:
error_handler.handle(ex, user_id=following)
else:
print('User already visited. Skipping...')
print(num + 1, '/', num_followings)
yield path_to
def target_follower_bfs(self, start_user):
"""
Same as origin_follower_bfs, but the other way round.
If user A and user B follows target, and user C follows user B:
C ---> B ---> target
A --->
:param start_user: This refers to the target user. The user from which to start the search.
:yield: All paths possible up to the current radius, in format set({user_id: path_to_that_user_id}, etc.)
"""
api = self.api
error_handler = ErrorHandler(api=api)
queue = Queue()
queue.put(start_user)
visited = {start_user}
path_to = defaultdict(lambda: [start_user])
_ = path_to[start_user]
for current_radius in range(self.search_radius):
if queue.qsize() == 0:
print('Queue empty. Ending target BFS.')
return
current_user = queue.get()
if self.cache.read_from_cache(
user_id=current_user
) is None: # current_user does not yet exist in cache
current_user_followers = api.GetFollowerIDs(
user_id=current_user)
current_user_screen_name = api.GetUser(
user_id=current_user).name
self.cache.append_to_cache(
user_id=current_user,
screen_name=current_user_screen_name,
follower_ids=current_user_followers)
print(
f"Target BFS: requested {current_user_screen_name}'s screen name and followers from API. "
f"Commencing search.")
else:
cached_data = self.cache.read_from_cache(user_id=current_user)
current_user_screen_name = cached_data[
'screen_name'] # if user_id exists as a key in cache,
# screen_name will always exist as well
if not cached_data[
'followers']: # i.e. follower list does not yet exist in cache
current_user_followers = api.GetFollowerIDs(
user_id=current_user)
self.cache.append_to_cache(
user_id=current_user,
screen_name=current_user_screen_name,
follower_ids=current_user_followers)
print(
f"Target BFS: retrieved {current_user_screen_name}'s screen name from cache. "
f"Requested followers from API. Commencing search.")
else:
current_user_followers = cached_data['followers']
print(
f"Target BFS: retrieved {current_user_screen_name}'s screen name AND followers from cache. "
f"Today is a good day. Commencing search.")
num_followers = len(current_user_followers)
for num, follower in enumerate(current_user_followers):
if follower not in visited:
try:
if self.cache.read_from_cache(
user_id=follower
) is None: # this user does not yet exist in cache
follower_screen_name = api.GetUser(
user_id=follower).name
self.cache.append_to_cache(
user_id=follower,
screen_name=follower_screen_name)
print(
f"Target BFS: {current_user_screen_name} is followed by {follower_screen_name}. "
f"Requested {follower_screen_name}'s screen name from API and cached."
)
else:
follower_screen_name = self.cache.read_from_cache(
user_id=follower)['screen_name']
print(
f"Target BFS: {current_user_screen_name} is followed by {follower_screen_name}. "
f"Retrieved {follower_screen_name}'s screen name from cache."
)
visited.add(follower)
queue.put(follower)
current_path = list(path_to[current_user])
current_path.extend([follower])
path_to[follower] = current_path
except twitter.error.TwitterError as ex:
error_handler.handle(ex, user_id=follower)
else:
print('User already visited. Skipping...')
print(num + 1, '/', num_followers)
yield path_to
def find_middle_users(self, paths_left, paths_right):
"""
Find intersection between two paths, i.e. the 'middle users'
:param paths_left:
:param paths_right:
:return: List of middle users
"""
middle_users = paths_left.keys() & paths_right.keys()
return middle_users
def reduce_path(self, paths_left, paths_right, middle_user):
"""
Reduce path from path from the left to path to the right, given the middle element
:param paths_left:
:param paths_right:
:param middle_user:
:return: Reduced paths
"""
possible_paths = []
for middle_user in middle_user:
new_path = paths_left[middle_user]
new_path.extend(list(reversed(paths_right[middle_user]))[1:])
possible_paths.append(new_path)
return possible_paths
def display_paths(self, paths):
"""
Simple UI method that prints the path out in English
:param paths: list of list of paths. will print them all.
"""
for path in paths:
for idx, user in enumerate(path):
screen_name = self.cache.read_from_cache(
user_id=user)['screen_name']
if idx == 0:
print(f'{screen_name} follows ', end='')
elif idx == len(path) - 1:
print(screen_name)
else:
print(f'{screen_name}, who follows ', end='')
def run_search(self, origin_user, target_user):
"""
Driver code for the whole class.
:param origin_user: The user to find a connection from
:param target_user: The user to find a connection to
:return: Possible paths in a list of lists of Twitter IDs, in X following Y order
"""
for idx, (paths_left, paths_right) in enumerate(
zip(self.origin_following_bfs(start_user=origin_user),
self.target_follower_bfs(start_user=target_user))):
print(
f"{len(paths_left)} total path(s) from origin. {len(paths_right)} total path(s) from target."
)
middle_users = self.find_middle_users(paths_left, paths_right)
if len(middle_users) != 0:
possible_paths = self.reduce_path(paths_left, paths_right,
middle_users)
print(f"{len(possible_paths)} possible path(s) found:")
self.display_paths(possible_paths)
else:
print("No possible paths found.")
if idx == self.search_radius - 1:
input(
"Search concluded with set search_radius. Press enter to close."
)
return possible_paths
if not self.auto_next:
input("Press enter to go to next radius")
