def restart(timestamp): """ Dumps data for the reachable nodes into a JSON file. Loads all reachable nodes from Redis into the crawl set. Removes keys for all nodes from current crawl. Updates excluded networks with current list of bogons. Updates number of reachable nodes and most common height in Redis. """ redis_pipe = REDIS_CONN.pipeline() nodes = REDIS_CONN.smembers('up') # Reachable nodes redis_pipe.delete('up') for node in nodes: (address, port, services) = node[5:].split("-", 2) redis_pipe.sadd('pending', (address, int(port), int(services))) for key in get_keys(REDIS_CONN, 'node:*'): redis_pipe.delete(key) for key in get_keys(REDIS_CONN, 'crawl:cidr:*'): redis_pipe.delete(key) redis_pipe.execute() update_excluded_networks() reachable_nodes = len(nodes) logging.info("Reachable nodes: %d", reachable_nodes) REDIS_CONN.lpush('nodes', (timestamp, reachable_nodes)) height = dump(timestamp, nodes) REDIS_CONN.set('height', height) logging.info("Height: %d", height)
def main(argv): test_conn() if len(argv) < 3 or not os.path.exists(argv[1]): print("Usage: crawl.py [config] [master|slave]") return 1 # Initialize global conf init_conf(argv) # Initialize logger loglevel = logging.INFO if CONF['debug']: loglevel = logging.DEBUG logformat = ("[%(process)d] %(asctime)s,%(msecs)05.1f %(levelname)s " "(%(funcName)s) %(message)s") logging.basicConfig(level=loglevel, format=logformat, filename=CONF['logfile'], filemode='a') print("Log: {}, press CTRL+C to terminate..".format(CONF['logfile'])) global REDIS_CONN REDIS_CONN = new_redis_conn(db=CONF['db']) if CONF['master']: REDIS_CONN.set('crawl:master:state', "starting") REDIS_CONN.set('crawl:master:blockchain', CONF['BLOCKCHAIN']) logging.info("Removing all keys") redis_pipe = REDIS_CONN.pipeline() for b in all_chains: redis_pipe.delete('up-{}'.format(b)) redis_pipe.delete('up') for key in get_keys(REDIS_CONN, 'node:*'): redis_pipe.delete(key) for key in get_keys(REDIS_CONN, 'crawl:cidr:*'): redis_pipe.delete(key) redis_pipe.delete('pending') redis_pipe.execute() set_pending() update_excluded_networks() REDIS_CONN.set('crawl:master:state', "running") # Spawn workers (greenlets) including one worker reserved for cron tasks workers = [] if CONF['master']: workers.append(gevent.spawn(cron)) for _ in xrange(CONF['workers'] - len(workers)): workers.append(gevent.spawn(task)) logging.info("Workers: %d", len(workers)) gevent.joinall(workers) return 0
def main(argv): if len(argv) < 3 or not os.path.exists(argv[1]): print("Usage: crawl.py [config] [master|slave]") return 1 # Initialize global conf init_conf(argv) # Initialize logger loglevel = logging.INFO if CONF['debug']: loglevel = logging.DEBUG logformat = ("[%(process)d] %(asctime)s,%(msecs)05.1f %(levelname)s " "(%(funcName)s) %(message)s") logging.basicConfig(level=loglevel, format=logformat, filename=CONF['logfile'], filemode='a') print("Log: {}, press CTRL+C to terminate..".format(CONF['logfile'])) global REDIS_CONN REDIS_CONN = new_redis_conn(db=CONF['db']) if CONF['master']: REDIS_CONN.set('crawl:master:state', "starting") logging.info("Removing all keys") redis_pipe = REDIS_CONN.pipeline() redis_pipe.delete('up') for key in get_keys(REDIS_CONN, 'node:*'): redis_pipe.delete(key) for key in get_keys(REDIS_CONN, 'crawl:cidr:*'): redis_pipe.delete(key) redis_pipe.delete('pending') redis_pipe.execute() set_pending() update_excluded_networks() REDIS_CONN.set('crawl:master:state', "running") # Spawn workers (greenlets) including one worker reserved for cron tasks workers = [] if CONF['master']: workers.append(gevent.spawn(cron)) for _ in xrange(CONF['workers'] - len(workers)): workers.append(gevent.spawn(task)) logging.info("Workers: %d", len(workers)) gevent.joinall(workers) return 0
def recovery_with_one_node_option(testparams, session, nodes, dropped_groups, indexes): """Test case for recovery with `--one-node` option.""" result = copy.deepcopy(recovery_skeleton) available_nodes = [node for node in nodes if node.group not in dropped_groups] node = random.choice(available_nodes) keys = utils.get_keys(testparams, session, dropped_groups, indexes) # Split inconsistent keys to recovered and still inconsistent keys # depend on following condition: "is the key belong to chosen node?" node_address = socket.gethostbyname(node.host) node_recovered_keys = {} node_inconsistent_keys = {} for key, key_indexes in keys["inconsistent"].items(): key_id = session.transform(key) key_node = session.lookup_address(key_id, node.group) if node_address == key_node.host and \ node.port == key_node.port: node_recovered_keys[key] = key_indexes else: node_inconsistent_keys[key] = key_indexes result["keys"]["consistent"] = keys["consistent"] result["keys"]["recovered"] = node_recovered_keys result["keys"]["inconsistent"] = node_inconsistent_keys result["recovery_indexes"] = False result["cmd"] = ["dnet_recovery", "--one-node", "{}:{}:2".format(node.host, node.port), "--groups", ','.join([str(g) for g in session.groups]), "dc"] return result
def __init__(self, args): self.args = args AWS_ACCESS_KEY, AWS_SECRET_KEY = get_keys(args.file_key) s3client = boto3.client('s3', aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY) self.s3client = s3client
def name_isexit(): while True: keys = get_keys() for key in keys: dict1 = get_values(key) write_log(str(dict1)) try: copywriting = dict1[b'copywriting'].decode('utf-8') rooms = [dict1[b'room'].decode('utf-8')] except: pass try: photo_path = dict1[b'photo_path'].decode('utf-8') if photo_path: send_photo(copywriting, photo_path, rooms) del_key(key) except: pass try: text = dict1[b'text'].decode('utf-8') if text: send_text(copywriting, text, rooms) del_key(key) except: pass time.sleep(60)
def output_reducer(self, label_key, pix_datas): """ fill the data in to a raster image and return the names of the generated images """ # download a template raster job = os.environ.get('LT_JOB') rast_keys = utils.get_keys(s.IN_RASTS % job) tmplt_key = [ k.key for k in rast_keys if s.RAST_TRIGGER in k.key ][0].key tmplt_rast = utils.rast_dl(tmplt_key) # name raster so it uploads to correct location rast_key = s.OUT_RAST_KEYNAME % (job, label_key) rast_fn = utils.keyname2filename(rast_key) # write data to raster utils.data2raster(pix_datas, tmplt_rast, out_fn=rast_fn) # upload raster rast_key = utils.upload([rast_fn])[0] yield label_key, [rast_key.key]
def setup_mapper(self, _, line): """ Reads in a dummy line from the input.txt file, ignores it, and sets up the job passed to MRLandTrendrJob by reading from the input S3 dir for that job. Outputs a list of the S3 keys for each of the input rasters """ job = os.environ.get('LT_JOB') print 'Setting up %s' % job rast_keys = [k.key for k in utils.get_keys(s.IN_RASTS % job)] analysis_rasts = filter(lambda k: s.RAST_SUFFIX in k, rast_keys) if not analysis_rasts: raise Exception('No analysis rasters specified for job %s' % job) # download template rast for grid rast_fn = utils.rast_dl(analysis_rasts[0]) # set up grid grid_fn = utils.keyname2filename(s.OUT_GRID % job) utils.rast2grid(rast_fn, out_csv=grid_fn) utils.upload([grid_fn]) # note - must yield at end to ensure grid is created for i, keyname in enumerate(analysis_rasts): yield i, keyname
def setup_mapper(self, _, line): """ Reads in a dummy line from the input.txt file, ignores it, and sets up the job passed to MRLandTrendrJob by reading from the input S3 dir for that job. Outputs a list of the S3 keys for each of the input rasters """ job = os.environ.get('LT_JOB') print 'Setting up %s' % job analysis_rasts = [ k.key for k in utils.get_keys(s.IN_RASTS % job) if s.RAST_TRIGGER in k.key ] if not analysis_rasts: raise Exception('No analysis rasters specified for job %s' % job) # download template rast for grid rast_fn = utils.rast_dl(analysis_rasts[0]) # set up grid grid_fn = utils.keyname2filename(s.OUT_GRID % job) utils.rast2grid(rast_fn, out_csv=grid_fn) utils.upload([grid_fn]) # note - must yield at end to ensure grid is created for i, keyname in enumerate(analysis_rasts): yield i, keyname
def recovery_with_dump_file_option(testparams, session, nodes, dropped_groups, indexes): """Test case for recovery with `--dump-file` option.""" result = copy.deepcopy(recovery_skeleton) keys = utils.get_keys(testparams, session, dropped_groups, indexes) # Split inconsistent keys: keys which will be recovered and keys which will not be inconsistent_keys_number = int(len(keys["inconsistent"]) * testparams["inconsistent_files_percentage"]) inconsistent_keys = dict(keys["inconsistent"].items()[:inconsistent_keys_number]) recovered_keys = dict(keys["inconsistent"].items()[inconsistent_keys_number:]) result["keys"]["consistent"] = keys["consistent"] result["keys"]["recovered"] = recovered_keys result["keys"]["inconsistent"] = inconsistent_keys result["recovery_indexes"] = False dump_file_path = "./dump_file" utils.dump_keys_to_file(session, result["keys"]["recovered"], dump_file_path) node = random.choice(nodes) result["cmd"] = ["dnet_recovery", "--remote", "{}:{}:2".format(node.host, node.port), "--groups", ','.join([str(g) for g in session.groups]), "--dump-file", dump_file_path, "dc"] return result
async def change_access_token(cls): if cls.last_updated: if datetime.now() - cls.last_updated < timedelta(0, 60, 0): return True query = "?grant_type=client_credentials" \ + "&client_id={}".format(utils.get_keys()["blizzard"]["id"]) \ + "&client_secret={}".format(utils.get_keys()["blizzard"]["secret"]) url = encode(cls.OAUTH_BASE, query) async with aiohttp.ClientSession() as session: async with session.get(url) as response: if response.status == 200: token = await response.json() cls.ACCESS_TOKEN = token["access_token"] cls.last_updated = datetime.now() return True else: return False
def main(argv): if len(argv) < 3 or not os.path.exists(argv[1]): print("Usage: crawl.py [config] [master|slave]") return 1 # Initialize global settings init_settings(argv) # Initialize logger loglevel = logging.INFO if SETTINGS['debug']: loglevel = logging.DEBUG logformat = ("[%(process)d] %(asctime)s,%(msecs)05.1f %(levelname)s " "(%(funcName)s) %(message)s") logging.basicConfig(level=loglevel, format=logformat, filename=SETTINGS['logfile'], filemode='a') print("Writing output to {}, press CTRL+C to terminate..".format( SETTINGS['logfile'])) if SETTINGS['master']: REDIS_CONN.set('crawl:master:state', "starting") logging.info("Removing all keys") redis_pipe = REDIS_CONN.pipeline() for key in get_keys(REDIS_CONN, 'node:*'): redis_pipe.delete(key) for key in get_keys(REDIS_CONN, 'crawl:cidr:*'): redis_pipe.delete(key) redis_pipe.delete('pending') redis_pipe.execute() set_pending() update_excluded_networks() # Spawn workers (greenlets) including one worker reserved for cron tasks workers = [] if SETTINGS['master']: workers.append(gevent.spawn(cron)) for _ in xrange(SETTINGS['workers'] - len(workers)): workers.append(gevent.spawn(task)) logging.info("Workers: %d", len(workers)) gevent.joinall(workers) return 0
def restart(timestamp): """ Dumps data for the reachable nodes into a JSON file. Loads all reachable nodes from Redis into the crawl set. Removes keys for all nodes from current crawl. Updates excluded networks with current list of bogons. Updates number of reachable nodes and most common height in Redis. """ redis_pipe = REDIS_CONN.pipeline() nodes = REDIS_CONN.smembers('up') # Reachable nodes redis_pipe.delete('up') for node in nodes: (address, port, services) = node[5:].split("-", 2) redis_pipe.sadd('pending', (address, int(port), int(services))) for key in get_keys(REDIS_CONN, 'node:*'): redis_pipe.delete(key) for key in get_keys(REDIS_CONN, 'crawl:cidr:*'): redis_pipe.delete(key) if CONF['include_checked']: checked_nodes = REDIS_CONN.zrangebyscore( 'check', timestamp - CONF['max_age'], timestamp) for node in checked_nodes: (address, port, services) = eval(node) if is_excluded(address): logging.debug("Exclude: %s", address) continue redis_pipe.sadd('pending', (address, port, services)) redis_pipe.execute() update_excluded_networks() reachable_nodes = len(nodes) logging.info("Reachable nodes: %d", reachable_nodes) REDIS_CONN.lpush('nodes', (timestamp, reachable_nodes)) height = dump(timestamp, nodes) REDIS_CONN.set('height', height) logging.info("Height: %d", height)
def restart(timestamp): """ Dumps data for the reachable nodes into a JSON file. Loads all reachable nodes from Redis into the crawl set. Removes keys for all nodes from current crawl. Updates excluded networks with current list of bogons. Updates number of reachable nodes and most common height in Redis. """ nodes = [] # Reachable nodes redis_pipe = REDIS_CONN.pipeline() for key in get_keys(REDIS_CONN, 'node:*'): state = REDIS_CONN.hget(key, 'state') if state == "up": nodes.append(key) (address, port, services) = key[5:].split("-", 2) redis_pipe.sadd('pending', (address, int(port), int(services))) redis_pipe.delete(key) for key in get_keys(REDIS_CONN, 'crawl:cidr:*'): redis_pipe.delete(key) if CONF['include_checked']: checked_nodes = REDIS_CONN.zrangebyscore('check', timestamp - CONF['max_age'], timestamp) for node in checked_nodes: (address, port, services) = eval(node) if is_excluded(address): logging.debug("Exclude: %s", address) continue redis_pipe.sadd('pending', (address, port, services)) redis_pipe.execute() update_excluded_networks() reachable_nodes = len(nodes) logging.info("Reachable nodes: %d", reachable_nodes) REDIS_CONN.lpush('nodes', (timestamp, reachable_nodes)) height = dump(timestamp, nodes) REDIS_CONN.set('height', height) logging.info("Height: %d", height)
def get_quote(symbol): if symbol.lower() == 'futures': return get_index_futures() # url = "https://api.iextrading.com/1.0/stock/"+symbol+"/quote?displayPercent=true" token = utils.get_keys("iex")['public'] url = "https://cloud.iexapis.com/stable/stock/%s/quote?displayPercent=true&token=%s" % ( symbol, token) print(url) req = Request(url) req.headers["User-Agent"] = "windows 10 bot" # Load data quote = json.loads(urlopen(req).read().decode("utf-8")) try: change = float(quote['change']) quote['ch'] = "%0.2f" % (change) quote['chper'] = "%0.2f" % (quote['changePercent']) quote['chytd'] = "%0.2f" % (quote['ytdChange']) except TypeError: change = "n/a" quote['ch'] = "n/a" quote['chper'] = "n/a" quote['chytd'] = "n/a" mcap = quote['marketCap'] if mcap is not None: if mcap >= 1e12: cap = round(mcap / 1e12, 1) cap = str(cap) + "T" elif mcap >= 1e9: cap = round(mcap / 1e9, 1) cap = str(cap) + "B" elif mcap >= 1e6: cap = round(mcap / 1e6, 1) cap = str(cap) + "M" elif mcap >= 1e3: cap = str(round(mcap / 1e3, 1)) + "k" else: cap = str(mcap) else: cap = "" if change != "n/a" and change > 0: quote['ch'] = "+" + quote['ch'] quote['chper'] = "+" + quote['chper'] output = "%s - %s:\n" % (symbol.upper(), quote['companyName']) output += "```python\n" if quote['latestSource'] != "IEX real time price": output += "Realtime price: {iexRealtimePrice}\n".format_map(quote) output += "{latestSource} - {latestTime}:\n" \ "Last price: {latestPrice} ({ch}, {chper}%, {chytd}% YTD)".format_map(quote) output = output + " %s mkt cap\n" % cap if quote['week52High'] is not None: output = output + "52w high: %.02f\t52w low:%.02f" % ( quote.get('week52High'), quote.get('week52Low')) output = output + "```" return output
async def get_reports(cls, region, realm, guild_name): query = "?api_key={}".format(utils.get_keys()["warcraftlogs"]["token"]) url = encode( "{}/reports/guild/{}/{}/{}".format(cls.BASE, guild_name, realm, region), query) async with aiohttp.ClientSession() as session: async with session.get(url) as response: if response.status == 200: return await response.json() else: return None
async def edit_poll(self, poll_info, message, pollObject): poll = self.embed( title=poll_info["question"], description=utils.to_str(pollObject.perfect_options(utils.get_keys(poll_info["options"]), poll_info)), colour=0x0892d0 # how to improve: make a poll object when poll is called in commands, making the poll obj have an attribute of get_question() and get_option() ) poll.set_footer(text=self.cfg["footer"]) await message.edit(content=None, embed=poll)
def default_recovery(testparams, session, nodes, dropped_groups, indexes): """Test case for recovery with no special options.""" result = copy.deepcopy(recovery_skeleton) keys = utils.get_keys(testparams, session, dropped_groups, indexes) result["keys"]["consistent"] = keys["consistent"] result["keys"]["recovered"] = keys["inconsistent"] node = random.choice(nodes) result["cmd"] = ["dnet_recovery", "--remote", "{}:{}:2".format(node.host, node.port), "--groups", ','.join([str(g) for g in session.groups]), "dc"] return result
async def get_rankings(cls, region, class_name, difficulty, encounter): class_id, spec_id = WCL_CLASS_IDS[class_name] metric = "hps" if CLASS_SPECS[class_name] in CLASS_HEALS else "dps" query = "?api_key={}".format(utils.get_keys()["warcraftlogs"]["token"]) \ + "®ion={}".format(region) \ + "&class={}".format(class_id) \ + "&spec={}".format(spec_id) \ + "&difficulty={}".format(difficulty) \ + "&metric={}".format(metric) url = encode("{}/rankings/encounter/{}".format(cls.BASE, encounter), query) async with aiohttp.ClientSession() as session: async with session.get(url) as response: if response.status == 200: return await response.json() else: return None
def get_quote(symbol): # url = "https://api.iextrading.com/1.0/stock/"+symbol+"/quote?displayPercent=true" token = utils.get_keys("iex")['public'] url = "https://cloud.iexapis.com/stable/stock/%s/quote?displayPercent=true&token=%s" % ( symbol, token) print(url) req = Request(url) req.headers["User-Agent"] = "windows 10 bot" # Load data quote = json.loads(urlopen(req).read().decode("utf-8")) try: change = float(quote['change']) ch = "%0.2f" % (change) chper = "%0.2f" % (quote['changePercent']) chytd = "%0.2f" % (quote['ytdChange']) except TypeError: change = "n/a" ch = "n/a" chper = "n/a" chytd = "n/a" mcap = quote['marketCap'] if mcap >= 1e12: cap = round(mcap / 1e12, 1) cap = str(cap) + "T" elif mcap >= 1e9: cap = round(mcap / 1e9, 1) cap = str(cap) + "B" elif mcap >= 1e6: cap = round(mcap / 1e6, 1) cap = str(cap) + "M" elif mcap >= 1e3: cap = str(round(mcap / 1e3, 1)) + "k" else: cap = str(mcap) if change != "n/a" and change > 0: ch = "+" + ch chper = "+" + chper output = "%s - %s:```python\n Last price: %s (%s, %s%%, %s%% YTD" % \ (symbol.upper(),quote['companyName'],quote['latestPrice'],ch,chper,chytd)+")" output = output + " %s mkt cap\n" % cap output = output + " 52w high: %.02f\t52w low:%.02f" % (quote['week52High'], quote['week52Low']) output = output + "```" return output
def get_current_weatherbit(text): url = "https://api.weatherbit.io/v2.0/current" key = utils.get_keys("weatherbit")['key'] lat,lon,loc = get_lat_lon(text) url = url + "?key=%s&units=I&lat=%f&lon=%f" % (key, lat, lon) resp = utils.get_json(url) if len(resp['data']) > 0: data = resp['data'][0] obstime = data['ob_time'] obsdatetime = datetime.strptime(obstime, '%Y-%m-%d %H:%M') ret = "```python\n" \ "Location search result: %s\n" \ "%s in %s, %s, %s:\n" \ " Temp: %.1f (Feels like: %.1f)\n" \ " Humidity: %d%%\n" \ " Observed: %s at station: %s```" % (loc, data['weather']['description'], data['city_name'], data['state_code'], data['country_code'], data['temp'], data['app_temp'], data['rh'], utils.prettydate(obsdatetime, utc=True), data['station']) return ret
def search_gfys(query, num=0): api_url = "https://api.gfycat.com/v1/" oauth_url = api_url + "oauth/token" search_url = api_url + "me/gfycats/search" keys = utils.get_keys('gfycat') params = { 'client_id': keys['client_id'], 'client_secret': keys['client_secret'], 'username': keys['username'], 'password': keys['password'], "grant_type": 'password' } # print(params) r = requests.post(oauth_url, data=str(params)) # print(r.json()) access_token = r.json()['access_token'] headers = {'Authorization': 'Bearer {}'.format(access_token)} url = search_url + "?search_text=%s&count=20" % urllib.parse.quote_plus( query) s = requests.get(url, headers=headers).json() try: if len(s['gfycats']) > 0: if num == 0: return gfy_str(s['gfycats'][0]) else: out = "" for i in range(min(num, len(s['gfycats']))): out = "%s%s\n" % (out, gfy_str(s['gfycats'][i], embed=False)) return out else: print("query: %s\n%s\n\n" % (query, s)) return "No gfycats found" except: import traceback f = utils.write_to_file(json.dumps(s), "gfycat.json", 'errors', prependtimestamp=True) print("error: json output written to %s" % f) traceback.print_exc() return "encountered error"
def get_indexable_bills(): revision = select([bill_revs,bills],bill_revs.c.bill_id==bills.c.id).apply_labels() conn = engine.connect() result = conn.execute(revision) data = result.fetchall() for item in data: temp = {} for key in item.keys()[1:]: new_key = utils.get_keys(key) if new_key == 'url': full_path = download(item[key]) if not full_path: continue temp['document'] = pyes.file_to_attachment(full_path) else: temp[new_key] = item[key] yield temp
def remove_vote(self, msg_id, member_name, vote_id): #shit this as well rip poll = self.load_polls()[msg_id] option_keys = utils.get_keys(poll["options"]) if vote_id == -1 and len( poll["options"] ) > 11: #number is the amount of available emojis that connect to one option for i in option_keys[10:]: #the number -1 if member_name in poll["options"][i]["names"]: poll["options"][i]["votes"] -= 1 poll["options"][i]["names"].remove(member_name) else: #i am cringing at how messy this is.. oof if member_name in poll["options"][option_keys[vote_id]]["names"]: poll["options"][option_keys[vote_id]]["votes"] -= 1 poll["options"][option_keys[vote_id]]["names"].remove( member_name) whole_poll = self.load_polls() whole_poll[msg_id] = poll self.save_polls(whole_poll)
def main(argv): if len(argv) < 3 or not os.path.exists(argv[1]): print("Usage: ping.py [config] [master|slave]") return 1 # Initialize global conf init_conf(argv) # Initialize logger loglevel = logging.INFO if CONF['debug']: loglevel = logging.DEBUG logformat = ("%(filename)s %(lineno)d %(levelname)s " "(%(funcName)s) %(message)s") logging.basicConfig(level=loglevel, format=logformat, filename=CONF['logfile'], filemode='a') print("Log: {}, press CTRL+C to terminate..".format(CONF['logfile'])) global REDIS_CONN REDIS_CONN = new_redis_conn(db=CONF['db']) if CONF['master']: redis_pipe = REDIS_CONN.pipeline() logging.info("Removing all keys") redis_pipe.delete('reachable') redis_pipe.delete('open') redis_pipe.delete('opendata') for key in get_keys(REDIS_CONN, 'ping:cidr:*'): logging.info("Deleting %s", key) redis_pipe.delete(key) redis_pipe.execute() # Initialize a pool of workers (greenlets) pool = gevent.pool.Pool(CONF['workers']) pool.spawn(cron, pool) pool.join() return 0
def main(argv): if len(argv) < 3 or not os.path.exists(argv[1]): print("Usage: ping.py [config] [master|slave]") return 1 # Initialize global conf init_conf(argv) # Initialize logger loglevel = logging.INFO if CONF['debug']: loglevel = logging.DEBUG logformat = ("[%(process)d] %(asctime)s,%(msecs)05.1f %(levelname)s " "(%(funcName)s) %(message)s") logging.basicConfig(level=loglevel, format=logformat, filename=CONF['logfile'], filemode='a') print("Log: {}, press CTRL+C to terminate..".format(CONF['logfile'])) global REDIS_CONN REDIS_CONN = new_redis_conn(db=CONF['db']) if CONF['master']: redis_pipe = REDIS_CONN.pipeline() logging.info("Removing all keys") redis_pipe.delete('reachable') redis_pipe.delete('open') redis_pipe.delete('opendata') for key in get_keys(REDIS_CONN, 'ping:cidr:*'): redis_pipe.delete(key) redis_pipe.execute() # Initialize a pool of workers (greenlets) pool = gevent.pool.Pool(CONF['workers']) pool.spawn(cron, pool) pool.join() return 0
def add_vote(self, msg_id, member_name, vote_id): #oh shit i gotta tidy this dumpsterfire up #sorry about the microsoft code below, to whoever is reading this poll = self.load_polls()[msg_id] option_keys = utils.get_keys(poll["options"]) if vote_id == -1 and len( poll["options"] ) > 11: #number is the amount of available emojis that connect to one option for i in option_keys[11:]: #the number -1 if member_name not in poll["options"][i]["names"]: poll["options"][i]["votes"] += 1 poll["options"][i]["names"].append(member_name) else: #i am cringing at how messy this is.. oof if member_name not in poll["options"][ option_keys[vote_id]]["names"]: poll["options"][option_keys[vote_id]]["votes"] += 1 poll["options"][option_keys[vote_id]]["names"].append( member_name) whole_poll = self.load_polls() whole_poll[msg_id] = poll self.save_polls(whole_poll)
def main(argv): if len(argv) < 3 or not os.path.exists(argv[1]): print("Usage: ping.py [config] [master|slave]") return 1 # Initialize global settings init_settings(argv) # Initialize logger loglevel = logging.INFO if SETTINGS['debug']: loglevel = logging.DEBUG logformat = ("[%(process)d] %(asctime)s,%(msecs)05.1f %(levelname)s " "(%(funcName)s) %(message)s") logging.basicConfig(level=loglevel, format=logformat, filename=SETTINGS['logfile'], filemode='a') print("Writing output to {}, press CTRL+C to terminate..".format( SETTINGS['logfile'])) if SETTINGS['master']: redis_pipe = REDIS_CONN.pipeline() logging.info("Removing all keys") redis_pipe.delete('reachable') redis_pipe.delete('open') redis_pipe.delete('opendata') for key in get_keys(REDIS_CONN, 'ping:cidr:*'): redis_pipe.delete(key) redis_pipe.execute() # Initialize a pool of workers (greenlets) pool = gevent.pool.Pool(SETTINGS['workers']) pool.spawn(cron, pool) pool.join() return 0
def get_stocks(): output = "Latest quotes:\n```python\n" stocks = [] token = utils.get_keys("iex")['public'] for symbol in ["DIA", "VOO", "VTI", "ONEQ", "VXUS"]: url = "https://cloud.iexapis.com/stable/stock/%s/quote?displayPercent=true&token=%s" % ( symbol, token) # url = "https://api.iextrading.com/1.0/stock/"+symbol+"/quote?displayPercent=true" req = Request(url) req.headers["User-Agent"] = "windows 10 bot" # Load data quote = json.loads(urlopen(req).read().decode("utf-8")) stock = dict() change = float(quote['change']) ch = "%0.2f" % (change) chper = "%0.2f" % (quote['changePercent']) chytd = "%0.2f" % (quote['ytdChange']) if change > 0: ch = "+" + ch chper = "+" + chper stock['symbol'] = symbol.upper() stock['price'] = "%.2f" % float(quote['latestPrice']) stock['change'] = ch stock['%'] = chper stock['% YTD'] = chytd stock['description'] = quote['companyName'] stock['high52w'] = quote['week52High'] stock['low52w'] = quote['week52Low'] stocks.append(stock) # output = output + "%s - %s (%s, %s%%, %s%% YTD) - %s\n" % (symbol.upper(),quote['latestPrice'],ch,chper,chytd,quote['companyName']) # output = "%s - %s:```python\n Last price: %s (%s, %s%%, %s%% YTD" % (symbol.upper(),quote['companyName'],quote['latestPrice'],ch,chper,chytd)+")" labels = ['symbol', 'price', 'change', '%', '% YTD', 'description'] left = ['symbol', 'description'] output = output + utils.format_table(labels, stocks, left_list=left) output = output + "```" return output
def output_reducer(self, label_key, pix_datas): """ fill the data in to a raster image and return the names of the generated images """ # download a template raster job = os.environ.get('LT_JOB') rast_keys = utils.get_keys(s.IN_RASTS % job) tmplt_key = filter(lambda x: s.RAST_SUFFIX in x.key, rast_keys)[0].key tmplt_rast = utils.rast_dl(tmplt_key) # name raster so it uploads to correct location rast_key = s.OUT_RAST_KEYNAME % (job, label_key) rast_fn = utils.keyname2filename(rast_key) # write data to raster utils.data2raster(pix_datas, tmplt_rast, out_fn=rast_fn) compressed = utils.compress([rast_fn], '%s.zip' % rast_fn) # upload raster rast_key = utils.upload([compressed])[0] yield label_key, [rast_key.key]
def output_reducer(self, label_key, pix_datas): """ fill the data in to a raster image and return the names of the generated images """ # download a template raster job = os.environ.get('LT_JOB') rast_keys = utils.get_keys(s.IN_RASTS % job) tmplt_key = [k.key for k in rast_keys if s.RAST_TRIGGER in k.key][0].key tmplt_rast = utils.rast_dl(tmplt_key) # name raster so it uploads to correct location rast_key = s.OUT_RAST_KEYNAME % (job, label_key) rast_fn = utils.keyname2filename(rast_key) # write data to raster utils.data2raster(pix_datas, tmplt_rast, out_fn=rast_fn) # upload raster rast_key = utils.upload([rast_fn])[0] yield label_key, [rast_key.key]
def restart(timestamp, start, elapsed): """ Dumps data for the reachable nodes into a JSON file. Loads all reachable nodes from Redis into the crawl set. Removes keys for all nodes from current crawl. Updates excluded networks with current list of bogons. Updates number of reachable nodes and most common height in Redis. EDIT this; not only json dump. also export to Postgres. """ #connection to Postgres Db: dbconn = psycopg2.connect( user="******", password="******", host="testpg", #host="pg_docker", #host='localhost', port="5432", database="btc_crawl") cursor = dbconn.cursor() redis_pipe = REDIS_CONN.pipeline() #nodes = REDIS_CONN.smembers('up') # Reachable nodes nodes = REDIS_CONN.smembers('up-{}'.format(CONF['BLOCKCHAIN'])) #redis_pipe.delete('up') redis_pipe.delete('up-{}'.format(CONF['BLOCKCHAIN'])) # Insert all nodes in all_nodes table; # Also, count them and make an insertion in master status - reachable for node in nodes: (address, port, services) = node[5:].split("-", 2) redis_pipe.sadd('pending', (address, int(port), int(services))) height_key = "height:{}-{}-{}-{}".format(address, port, services, CONF['BLOCKCHAIN']) height = 0 try: height = int(REDIS_CONN.get(height_key)) except TypeError: logging.warning("%s missing", height_key) height = 0 version = REDIS_CONN.get(node) glinks = REDIS_CONN.smembers('glinks-{}-{}'.format( address, CONF['BLOCKCHAIN'])) #JUST A REMINDER: #redis_pipe.sadd('glinks-{}-{}'.format(source_ip, CONF['BLOCKCHAIN']), '{}-{}-{}'.format( address, port, now)) #print("GLINKS LEN for node %s on chain %s: %d" %(address, CONF['BLOCKCHAIN'], len(glinks))) for link in glinks: l = link.split("-") #print(l) cursor.execute( "INSERT INTO GRAPH_LINKS " "(SOURCE_IP, SOURCE_PORT, BLOCKCHAIN, SINK_IP, SINK_PORT, FIRST_SEEN) " "VALUES (%s, %s, %s, %s, %s, %s) " "ON CONFLICT (source_ip, source_port, blockchain, sink_ip, sink_port) " "DO UPDATE SET " "last_seen=EXCLUDED.FIRST_SEEN, " "COUNTER_SEEN=COALESCE(graph_links.COUNTER_SEEN,0)+1", (str(address), port, CONF['BLOCKCHAIN'], str(l[0]), int( l[1]), int(l[2]))) dbconn.commit() redis_pipe.delete('glinks-{}-{}'.format(address, CONF['BLOCKCHAIN'])) cursor.execute( "INSERT INTO ALL_NODES " "(IP_ADDRESS, PORT, SERVICES, HEIGHT, FIRST_SEEN, VERSION, MY_IP, BLOCKCHAIN) " "VALUES(%s, %s, %s, %s, %s, %s, %s, %s) " "on conflict " "(IP_ADDRESS, PORT, services, blockchain) " "DO UPDATE set " "last_seen=excluded.FIRST_SEEN, " "height=excluded.height, " "version=excluded.version, " "COUNTER_SEEN=COALESCE(ALL_NODES.COUNTER_SEEN,0)+1", (str(address), port, str(services), height, timestamp, str(version), CONF['MY_IP'], CONF['BLOCKCHAIN'])) dbconn.commit() for key in get_keys(REDIS_CONN, 'node:*'): redis_pipe.delete(key) for key in get_keys(REDIS_CONN, 'crawl:cidr:*'): redis_pipe.delete(key) if CONF['include_checked']: checked_nodes = REDIS_CONN.zrangebyscore('check', timestamp - CONF['max_age'], timestamp) for node in checked_nodes: (address, port, services) = eval(node) if is_excluded(address): logging.debug("Exclude: %s", address) continue redis_pipe.sadd('pending', (address, port, services)) redis_pipe.execute() update_excluded_networks() reachable_nodes = len(nodes) logging.info("Reachable nodes: %d", reachable_nodes) REDIS_CONN.lpush('nodes', (timestamp, reachable_nodes)) height = dump(timestamp, nodes) REDIS_CONN.set('height-{}'.format(CONF['BLOCKCHAIN']), height) logging.info("(%s)_Height: %d" % (CONF['BLOCKCHAIN'], height)) cursor.execute( "INSERT INTO MASTER_STATUS (STATUS, START_TIME, ELAPSED_TIME, HEIGHT, BLOCKCHAIN, REACHABLE_NODES)" "VALUES(%s, %s, %s, %s, %s, %s)", ('Finished', start, elapsed, height, CONF['BLOCKCHAIN'], reachable_nodes)) dbconn.commit() cursor.close() dbconn.close() CONF['BLOCKCHAIN'] = cyclingBlockchainList.next() REDIS_CONN.set('crawl:master:blockchain', CONF['BLOCKCHAIN']) logging.info("Change Chain: %s", CONF['BLOCKCHAIN']) set_bchain_params() set_pending()
client = KerberosClient(HDFS_URL) conn = jdbc.connect("oracle.jdbc.driver.OracleDriver", URL_ORACLE_SERVER, [USER_ORACLE, PASSWD_ORACLE], ORACLE_DRIVER_PATH) curs = conn.cursor() df = get_train_data(curs, start_date=START_DATE, end_date=END_DATE) nb_documents = len(df) if nb_documents == 0: print('No data to train model!') sys.exit() else: print('{} documents available to train model.\n'.format(nb_documents)) train_keys = get_keys(df, ID_COLUMN) print('Preparing data...') df[TEXT_COLUMN] = df[TEXT_COLUMN].apply(clean_text) # Labels need to be grouped to be passed to the MultiLabelBinarizer df = df.groupby(TEXT_COLUMN)\ .agg(lambda x: set(x))\ .reset_index() classes = get_list_of_classes(curs) mlb = MultiLabelBinarizer(classes) y = df[LABEL_COLUMN] y = mlb.fit_transform(y) NEGATIVE_COLUMN_INDEX = np.where(mlb.classes_ == NEGATIVE_CLASS_VALUE)[0][0]
def test_get_keys(): expected_output = GET_KEYS_RESULT_EXAMPLE output = get_keys(EXPANDED_RESULT_EXAMPLE, 'SNCA_DK') assert output == expected_output
model_dates = sorted(client.list(FORMATTED_HDFS_PATH)) validated_datasets = [] classified_datasets = [] for model_date in model_dates: try: data_hdfs = get_results_from_hdfs(client, FORMATTED_HDFS_PATH, model_date=model_date) except BaseException: continue # Results are stored as a tuple represented as a string data_hdfs['MDEC_DK'] = data_hdfs['MDEC_DK'].apply( lambda x: ast.literal_eval(x)) keys = get_keys(data_hdfs, 'SNCA_DK') # Only needs the keys that are in the predictions data_oracle = get_evaluate_data(curs, keys) data_hdfs = expand_results(data_hdfs) # TODO: Optimization possible here, # gets all keys in query each time before filtering by keys sinalid_id_df = get_id_sinalid(curs, keys) data_hdfs = pd.merge(data_hdfs, sinalid_id_df, how='left', on='SNCA_DK') date = '{}/{}/{}'.format(model_date[6:8], model_date[4:6], model_date[:4]) data_hdfs['DT_MODELO'] = date data_oracle['DT_MODELO'] = date data_oracle['IS_VALIDATION'] = True
def buildModel(params): #asumming we're passed a dictionary of requisite #parameters, we can unpack it and construct a model #from the inputs. #bind the parameters locally for easier reference. ( MTT, #scalar, Total number of training teams ms, #numerical range of [0..MTT) WKS, #scalr, total number of weeks to schedule ws, #numerical range of [0..WKS) types, #set of unit types total_units, #scalar count of units us, #numerical range [0..total_units) unit_type, #unitid->type interval_type, #unit_type->training-interval #all the yearly demands. #yearly demands can be trivially translated to weekly #demands... msn_demand) = utils.get_keys(params, 'MTT', 'ms', 'WKS', 'ws', 'types', 'total_units', 'us', 'unit_type', 'interval_type', 'msn_demand') ## #setting up a minimization. sched = LpProblem("Sched", LpMinimize) months_weeks = 12 / 52 #scale the weeks to months.... ws = range(WKS * months_weeks) #scale the intervals to months.... interval_type = { k: weeks * months_weeks for k, weeks in interval_type.items() } #variables print(len(ws)) ## assign(m,u,w) # binary decision variable determining whether training ## team m is assigned to train unit u, during week w of the schedule. #This lets us say "we want to create family of variables, across the #index m_u_w, with similar properties (in this case they're binary) assign = LpVariable.dicts('assign', utils.product(ms, us, ws), lowBound=0.0, upBound=1.0, cat=LpInteger) #Objective function: z = assigncost = sum(m,u,w)assign(m,u,w) #min z, e.g. minimize the total number of assignments #For now, we reflect no higher order notion of cost within the parameters. #One could easily envision certain combinations being more desirable, #hence the inclusion of a weight parameter assigncost = LpVariable('assigncost') sched += assigncost == lpSum(assign[(m, u, w)] * 1.0 for (m, u, w) in utils.product(ms, us, ws)) #helper variables: #mtt training events per week #events(m,w) = sum(u)assign(m,u,w) forall m in ms,w in ws, u in us events = LpVariable.dicts("events", utils.product(ms, ws)) for (m, w) in events.keys(): sched += events[m, w] == lpSum((assign[m, u, w] for u in us)) #only one training event allowed per week for (m, w) in events.keys(): sched += events[m, w] <= 1 #add a var to track unit training trained = LpVariable.dicts("trained", utils.product(us, ws)) for (u, w) in trained.keys(): sched += trained[u, w] == lpSum((assign[m, u, w] for m in ms)) #unit training credits by interval #We account for training intervals by #recording wait times as a function of #max training intervals and previous #wait times. We introduce the linear #decision variable, wait(u,w), which #indicates how long unit u has been waiting #for training at week w. #we may want to add in some minimum frequency too, to ensure #we don't allow things like month-month training, which is likely #infeasible in practice. Something like, 1/2 the interval is the #lower bound. wait = LpVariable.dicts("wait", utils.product(us, ws), lowBound=0.0) for u in us: interval = interval_type[unit_type[u]] for w in utils.butlast(ws): sched += wait[u, w + 1] == wait[u, w] + 1 - trained[u, w] * (interval + 1) #we can train a unit more frequently, but not lapse in training. #Wait times cannot exceed mandated training intervals. #note: we can add a deviation goal to this later if we want #to or need to relax this. Alternately, include in objective... for (u, w) in wait.keys(): sched += wait[u, w] <= interval_type[unit_type[u]] #goal is to minimize the total number of assignments we need. #This should induce maximal wait times as well. sched.setObjective(assigncost) def getsolution(): res = {} for (m, u, w) in assign.keys(): if value(assign[m, u, w]) > 0.0: res[m, u, w] = value(assign[m, u, w]) return { 'assigned': res, #dictvals(assign), 'trained': dictvals(trained), 'wait': dictvals(wait), 'assigncost': value(assigncost) } return (sched, getsolution)
from flask import Flask, render_template, redirect, url_for, g, session, request import settings import utils from responses import resposta from database.objects import Conta, Pagamento, Usuario from authentication import authenticate, unauthenticate, authenticate_api from errors import KnownError, MissingParamsError, UserNotLoggedInError app = Flask(__name__, static_folder=settings.STATIC_FOLDER, template_folder=settings.TEMPLATES_FOLDER) app.secret_key = utils.get_keys()["APP_SECRET_KEY"] # import os; print(os.urandom(16)) @app.errorhandler(500) def handle_server_error(e): logger.exception(e) return "Um erro ocorreu no servidor" @app.route("/") @authenticate def page_index(): return redirect(url_for("page_resumo")) @app.route("/entrar") @unauthenticate def page_entrar(): return render_template("login.html")