def main(): db_connection = setup_db().connect() logger = Logger(db_connection) logger.info('Initializing Downloader') try: run_downloader(db_connection, logger) except Exception: logger.exception() logger.info('Finishing Downloader')
def worker(self, ip): ''' Make the post request for each record that we have in records list ''' for host in self.records: host['ipv4Address'] = ip Response = self.rest.post('/dns/' + str(host['id']), host) if Response == True: Logger.info(f"Updating IP address {ip} to Dynu.net record {host['name']}") else: Logger.Error(f"Failed to update IP address {ip} to Dynu.net record {host['name']}")
def create_path_if_not_exists(path): logger = Logger().getLogger() p = Path(path) if not p.exists(): try: p.mkdir(parents=True, exist_ok=True) except Exception as ex: logger.error("Creation of the directory %s failed" % path) logger.error(ex) else: logger.info("Successfully created the directory %s" % path)
class RiotExtractor: def __init__(self): self.logger = Logger().getLogger() def get_data_from_api(self, endpoint="", headers="", api_name=""): try: self.logger.info("getting {}'s data from {} with headers {}".format(api_name, endpoint, headers)) response = requests.request("GET", endpoint, headers=headers) if response.status_code != 200: raise ValueError('API dont work as expected, getting status code {} and error {}' .format(response.status_code, response.json())) return response.json() except Exception as ex: self.logger.error("getting {}'s data from {} with headers {}".format(api_name, endpoint, headers)) self.logger.error('Error: {}'.format(ex))
class Writer: def __init__(self): self.logger = Logger().getLogger() def write_json(self, data="", file_name="data.json", output_path=""): try: output_path = Utils.remove_slash_in_end_of_string(output_path) full_output_path = file_name + '.json' if output_path == "" else '{}/{}.json'.format( output_path, file_name) if data is not None: self.logger.info("Writing {} in json format".format(file_name)) with open(full_output_path, 'w') as f: json.dump(data, f) else: raise ValueError( 'file data is none, cannot write in file system') except Exception as ex: self.logger.error("can't write json in file system") self.logger.error('Error: {}'.format(ex))
class RiotChampionPointsXChampionID: def __init__(self, data): self.logger = Logger().getLogger() self.data = data self.chart = self.plot_chart() def plot_chart(self): try: self.logger.info("Plotting chart") # filter only championId and championPoints columns dataframe = pd.DataFrame( self.data)[['championId', 'championPoints']] dataframe['championId'] = dataframe['championId'].astype(str) # sort values to get the top 10 values descending dataframe = dataframe.sort_values(by=['championPoints'], ascending=False).head(10) chart = pygal.Bar(x_title='champion ID', y_title='champion Points') chart.title = 'Top 10 championPoints by championId' # adding data into chart object for index, row in dataframe.iterrows(): chart.add('id ' + row['championId'], row['championPoints']) self.logger.info("chart plotted") return chart except Exception as ex: self.logger.error("can't plot chart") self.logger.error('Error: {}'.format(ex)) def save_chart_as_image(self, path=""): try: path = Utils.remove_slash_in_end_of_string(path) path_full_png = path + '/champion_points_champion_id.png' path_full_svg = path + '/champion_points_champion_id.svg' self.chart.render_to_file(path_full_svg) self.chart.render_to_png(filename=path_full_png) self.logger.info("chart saved at {} as svg".format(path_full_svg)) self.logger.info("chart saved at {} as png".format(path_full_png)) except Exception as ex: self.logger.error("can't save chart in {}".format(path)) self.logger.error('Error: {}'.format(ex))
def upload_worker(queue, job_id, worker_id): # Uploader to the S3 bucket. db_connection = setup_db().connect() logger = Logger(db_connection, job_id) bucket_name = environ.get('UPLOAD_BUCKET') try: logger.info(f'Creating S3 uploader #{worker_id}') uploader = S3Uploader(bucket=bucket_name) granule_serializer = Serializer(db_connection, granule) while True: message = queue.get() if message == 'DONE': # Put in back for other workers. queue.put('DONE') break product_id, filename = message try: checksum = hashlib.md5(open(filename, 'rb').read()).hexdigest().upper() # Download status = SUCCESS granule_serializer.put( product_id, { 'download_status': DownloadStatus.SUCCESS, 'downloaded_at': datetime.now(), 'validated': False, 'checksum': checksum, 's3_location': f'{bucket_name}/filename' }) logger.info(f'Uploading {product_id} at #{worker_id}', f'Filename: {filename}') uploader.upload_file(filename) remove(filename) logger.info(f'Uploaded {product_id} at #{worker_id}', f'Filename: {filename}') except Exception: logger.exception() except Exception: logger.exception()
if zone == 'global': continue provider = config[zone].get('provider') try: module = __import__("plugins." + provider, fromlist=[provider]) except ImportError: Logger.error(f"Unknown provider in zone '{zone}': {provider}") zones[zone] = getattr(module, provider)(zone, config[zone]) ''' Get the update interval from the configuration file or fallback to default interval 300 seconds ''' update_interval_sec = config['global'].get('interval') if not update_interval_sec: update_interval_sec = 300 ip = None while True: new_ip = IP.getPublic() if not ip or ip != new_ip: ip = new_ip Logger.info(f"IP changed, updating zones/hosts with IP: {new_ip}") for _, zone_invoke in zones.items(): zone_invoke.worker(ip) time.sleep(int(update_interval_sec))
class Hemnet(): def __init__(self): self.log = Logger("Hemnet") self.request = Request() #Base objects for searches and results self.baseUrl = "http://www.hemnet.se" self.baseSearch = self.baseUrl + "/sok/create" self.baseLocation = self.baseUrl + "/locations/show?" self.baseResult = self.baseUrl + "/resultat" self.searchQuery = {} #Basetype, english -> Swedish self.translatedTypes = { "municipality": "Kommun", "district": u"Område", "postal_city": "Stadsdel", "region": u"Län", "street": "Gata", "city": "Stad" } #BaseAverageTypes -> Swedish self.translatedAverageTypes = { "age": u"List ålder", "price": "Medelpris", "price_m2": u"Pris per m²", "size": u"Storlek (m²)", "rooms": "Antal rum", "fee": u"Månadsavgift", "price_change_up": u"Prisökning (%)", "price_change_down": u"Prissäkning (%)" } #searchTypes self.searchTypes = { "f": "fritidshus", "v": "villa", "t": "tomt", "r": "radhus", "g": "gard", "b": "bostadsratt", "o": "other", "a": "all" } #Items to get average for self.itemAverageTypes = { "age": 0, "price": 0, "price_m2": 0, "size": 0, "rooms": 0, "fee": 0, "price_change_up": 0, "price_change_down": 0 } #Base result format self.resultFormat = { "totalItems": 0, "results": {} } self.log.info("Initiated Hemnet") ''' Searchdata is a formpost in a very specific format ''' def createSearchFormData(self, data, specificType='a'): locationData = [{ "id": (data.get("id")), "name": (data.get("name")), "parent_id": (data.get("parent_location").get("id")), "parent_name": (data.get("parent_location").get("name")) }] searchData = { "search[location_search]": locationData, "search[location_ids][]": data.get("id"), "search[region_id]": -1, "search[municipality_ids][]": -1, "search[country_id]": 0, "search[item_types][]": "%s" % self.searchTypes[specificType], "search[price_min]": '', "search[price_max]": '', "search[fee_max]": '', "search[rooms_min]": '', "search[living_area_min]": '', "search[keywords]": '', "commit": '' } return searchData def searchRequest(self, query): return self.request.postRequest(self.baseSearch, query) ''' Pass a list of keys and a dict of data to caluclate average value for each key ''' def avgByKey(self, keys, data): final = {} for d in data: for k in d.keys(): if k in keys: final[k] = final.get(k, 0) + d[k] for k in final.keys(): final[k] = final[k] / len(data) return final def getLocationQueryURL(self, query): return "%sq=%s" % (self.baseLocation, urllib.quote(query.encode('utf-8'))) @cache.methodcache.cache('findLocations', expire=72000) def findLocations(self, query, extra, area=None): queryURL = self.getLocationQueryURL(query) cacheResult = cache.locations.get(hashlib.md5(queryURL).hexdigest()) if (cacheResult is not None): print "Found cached loc" return cacheResult locFormData = [] locResponse = self.request.getResponse(queryURL, None) jdata = json.loads(locResponse) print json.dumps(jdata, indent=4) formData = {} locations = [] for id, item in enumerate(jdata): item["score"] = Levenshtein.ratio( item.get("location").get("name"), query) if (area is not None): if (item.get("location").get("parent_location").get( "name").find(area) != -1): formData = self.createSearchFormData( item.get("location"), extra) locations.append(item) locFormData.append(formData) else: formData = self.createSearchFormData(item.get("location"), extra) locations.append(item) locFormData.append(formData) locations = sorted(locations, key=itemgetter('score'), reverse=True) result = { 'search': locFormData, 'area': area, 'locations': locations } cache.locations[hashlib.md5(queryURL).hexdigest()] = result return result @cache.methodcache.cache('performSearch', expire=72000) def performSearch(self, searchData): hashkey = hashlib.md5(json.dumps(searchData, sort_keys=True)).hexdigest() cachedResult = cache.storage.get(hashkey) if (cachedResult is not None): print "Found cached searchResponse" return cachedResult print "Performing search on " + json.dumps(searchData, indent=4) searchRequest = self.searchRequest(searchData) searchResponse = self.request.getUnicodeDoc(searchRequest) resultData = self.parseResult(searchResponse, self.resultFormat) result = self.createResultItem(resultData) print "Storing hash " + hashkey chart_list = cache.storage.get(hashkey, {}) # metadata is the chart item minus the actual list plus a size metadata_keys = filter(lambda k: k != 'list', result.keys()) metadata = {key: result[key] for key in metadata_keys} chart_list[hashkey] = metadata cache.storage[hashkey] = chart_list[hashkey] return result def parseResult(self, doc, brokers={}): brokers = self.parseItems( doc.xpath("//div[contains(@class, 'item result')]"), brokers) nextpage = doc.xpath('//a[@class="next_page"]') try: url = nextpage[0].attrib["href"] if url is not None: self.log.info("Parsing %s" % url) nextDoc = self.request.requestUnicodeDoc(self.baseUrl + url) self.parseResult(nextDoc, brokers) except Exception, e: self.log.debug("ParseResult %s" % e) pass return brokers
class Hemnet() : def __init__(self): self.log = Logger("Hemnet"); self.request = Request(); #Base objects for searches and results self.baseUrl = "http://www.hemnet.se"; self.baseSearch = self.baseUrl + "/sok/create"; self.baseLocation = self.baseUrl + "/locations/show?"; self.baseResult = self.baseUrl + "/resultat"; self.searchQuery = {} #Basetype, english -> Swedish self.translatedTypes = { "municipality" : "Kommun", "district" : u"Område", "postal_city" : "Stadsdel", "region" : u"Län", "street" : "Gata", "city" : "Stad" } #BaseAverageTypes -> Swedish self.translatedAverageTypes = { "age" : u"List ålder", "price" : "Medelpris", "price_m2" : u"Pris per m²", "size" : u"Storlek (m²)", "rooms" : "Antal rum", "fee" : u"Månadsavgift", "price_change_up" : u"Prisökning (%)", "price_change_down" : u"Prissäkning (%)" } #searchTypes self.searchTypes = { "f" : "fritidshus", "v" : "villa", "t" : "tomt", "r" : "radhus", "g" : "gard", "b" : "bostadsratt", "o" : "other", "a" : "all" } #Items to get average for self.itemAverageTypes = { "age" : 0, "price" : 0, "price_m2" : 0, "size" : 0, "rooms" : 0, "fee" : 0, "price_change_up" : 0, "price_change_down" : 0 }; #Base result format self.resultFormat = { "totalItems" : 0, "results" : {} }; self.log.info("Initiated Hemnet"); ''' Searchdata is a formpost in a very specific format ''' def createSearchFormData(self, data, specificType = 'a') : locationData = [{ "id": (data.get("id")), "name": (data.get("name")), "parent_id": (data.get("parent_location").get("id")), "parent_name": (data.get("parent_location").get("name")) }] searchData = { "search[location_search]" : locationData, "search[location_ids][]": data.get("id"), "search[region_id]":-1, "search[municipality_ids][]":-1, "search[country_id]":0, "search[item_types][]": "%s" % self.searchTypes[specificType], "search[price_min]": '', "search[price_max]": '', "search[fee_max]": '', "search[rooms_min]": '', "search[living_area_min]": '', "search[keywords]":'', "commit": '' } return searchData; def searchRequest(self, query) : return self.request.postRequest(self.baseSearch, query); ''' Pass a list of keys and a dict of data to caluclate average value for each key ''' def avgByKey(self, keys, data): final = {} for d in data: for k in d.keys(): if k in keys: final[k] = final.get(k,0) + d[k] for k in final.keys(): final[k] = final[k]/len(data); return final; def getLocationQueryURL(self, query): return "%sq=%s" % (self.baseLocation, urllib.quote(query.encode('utf-8'))) @cache.methodcache.cache('findLocations', expire=72000) def findLocations(self, query, extra, area = None) : queryURL = self.getLocationQueryURL(query); cacheResult = cache.locations.get(hashlib.md5(queryURL).hexdigest()); if( cacheResult is not None): print "Found cached loc"; return cacheResult; locFormData = [] locResponse = self.request.getResponse(queryURL, None) jdata = json.loads(locResponse); print json.dumps(jdata, indent=4); formData = {} locations = [] for id, item in enumerate(jdata) : item["score"] = Levenshtein.ratio(item.get("location").get("name"), query) if( area is not None ): if( item.get("location").get("parent_location").get("name").find(area) != -1 ): formData = self.createSearchFormData(item.get("location"), extra); locations.append(item) locFormData.append(formData); else: formData = self.createSearchFormData(item.get("location"), extra); locations.append(item) locFormData.append(formData); locations = sorted(locations, key=itemgetter('score'), reverse=True) result = {'search' : locFormData, 'area' : area, 'locations' : locations }; cache.locations[hashlib.md5(queryURL).hexdigest()] = result return result; @cache.methodcache.cache('performSearch', expire=72000) def performSearch(self, searchData): hashkey = hashlib.md5( json.dumps(searchData, sort_keys=True) ).hexdigest(); cachedResult = cache.storage.get(hashkey); if(cachedResult is not None): print "Found cached searchResponse"; return cachedResult; print "Performing search on " + json.dumps(searchData, indent=4); searchRequest = self.searchRequest(searchData); searchResponse = self.request.getUnicodeDoc(searchRequest); resultData = self.parseResult(searchResponse, self.resultFormat); result = self.createResultItem(resultData); print "Storing hash " + hashkey; chart_list = cache.storage.get(hashkey, {}) # metadata is the chart item minus the actual list plus a size metadata_keys = filter(lambda k: k != 'list', result.keys()) metadata = { key: result[key] for key in metadata_keys } chart_list[hashkey] = metadata cache.storage[hashkey] = chart_list[hashkey] return result; def parseResult(self, doc, brokers = {}) : brokers = self.parseItems(doc.xpath("//div[contains(@class, 'item result')]"), brokers); nextpage = doc.xpath('//a[@class="next_page"]'); try: url = nextpage[0].attrib["href"]; if url is not None: self.log.info("Parsing %s" % url); nextDoc = self.request.requestUnicodeDoc(self.baseUrl + url); self.parseResult(nextDoc, brokers); except Exception,e: self.log.debug("ParseResult %s" % e) pass; return brokers;
def compare_files(file_set1, file_set2, data_folder_1, data_folder_2, config): # Print info about the compared files Logger.output("Directory1: {}\nDirectory2: {}".format( data_folder_1, data_folder_2)) # Make sure this is flushed before multiprocessing starts, otherwise # it may be written multiple times # Passing "flush=True" is not enough if the output target is a file Logger.flush_output() comparator = FilesetComparator(files1, files2, config) pairs = comparator.get_files_to_compare() # When sorting , every value has to be computed before starting printing delay_output = False if config.sort_order.lower() == "distance": delay_output = True elif config.sort_order.lower() == "path": delay_output = True # Build a partial func by passing config and delay output, so the result # can be used by pool.map func = partial(_compare, config, delay_output) # Use lock so lines aren't mixed up in output lock = multiprocessing.Lock() with multiprocessing.Pool(config.jobs, initializer=_init_process, initargs=(lock, )) as pool: edits = pool.map(func, pairs) edits = [edit for edit in edits if edit is not None] # If necessary, sort and then output the result Logger.progress("Generating output...") if config.sort_order.lower() == "distance": edits.sort(key=operator.itemgetter(2), reverse=True) elif config.sort_order.lower() == "path": edits.sort(key=operator.itemgetter(0), reverse=True) if delay_output: for edit in edits: output_change(edit, config) # Print info about the added and removed files added_count = 0 for added in comparator.added_files: added_count += 1 mime = "{} ({})".format(added.type["full"], added.type["mime"]) Logger.output("\nAdded: {}\nMime: {}".format(added.path, mime)) removed_count = 0 for removed in comparator.removed_files: removed_count += 1 mime = "{} ({})".format(removed.type["full"], removed.type["mime"]) Logger.output("\nRemoved: {}\nMime: {}".format(removed.path, mime)) # Print overall statistics Logger.info( "\nFound {} added files, {} removed files and {} changed files ({} files in total)" .format(added_count, removed_count, len(edits), added_count + removed_count + len(edits)))
except Exception: print( "Must turn debugger on at port {}. See https://www.jetbrains.com/help/idea/remote-debugging-with-product.html#remote-debug-config" .format(debuggerPort)) sys.exit() # initialize.PyfyApp().run() print("does this run now?") if args.symbol: symbol = args.symbol Logger.trace("Symbol passed in: {}".format(args.symbol)) quote = iex.getQuote(symbol) Logger.trace("Queried Symbol Quote is ...") Logger.trace(quote) if (quote != None): Logger.info('Current Price: {}'.format(quote.get('latestPrice'))) Logger.info('PE Ratio: {}'.format(quote.get('peRatio'))) Logger.info('52-Week High: {}'.format(quote.get('week52High'))) Logger.info('52-Week Low: {}'.format(quote.get('week52Low'))) else: Logger.error( 'Could not print symbol information for {}'.format(symbol)) if args.price_dates: parsedDate = dateparser.parse(args.graphprice) if (parsedDate.weekday() < 5): graphs.byMinuteIntradayLineGraph(symbol, 'average', parsedDate) else: Logger.all('Provided date is not a weekday, the day was: ' + parsedDate.strftime('%A'))
class ArbitrageProcess: logger = None open_order = False exchanges_one = [] exchanges_two = [] exchange_manager = None arbitrage_manager = None bot_manager = None arbitrage = None bot = None # TODO: maybe change to some config file BALANCE_LIMIT = 0.002 # minimum balance needed on the exchange def __init__(self, botId): self.logger = Logger("arbitrage") self.logger.info("#" + str(botId) + ": Initializing.") self.bot_manager = BotManager() self.exchange_manager = ExchangeManager() self.arbitrage_manager = ArbitrageManager() self.bot = self.bot_manager.get_bot(botId) self.arbitrage = self.arbitrage_manager.get_arbitrage(botId) def start_process(self): """ Start the arbitrage process by checking the balances on the exchanges first, then making a verdict on the prices and if viable place an order, repeat. """ print( strftime('%Y%m%d%H%M%S') + ' starting arbitrage process for bot#' + str(self.bot.get_id())) self.logger.info("#" + str(self.bot.get_id()) + ": Starting.") while not self.open_order: try: if not self.exchanges_one or not self.exchanges_two: self.init_exchanges() if self.check_balance(): verdict = self.do_magic() if verdict: self.place_orders(verdict) self.finish_bot() except Exception as e: print(strftime('%Y%m%d%H%M%S') + ' ' + str(e)) self.logger.info(str(e)) self.logger.info(traceback.format_exc()) # TODO: uncomment this line # self.bot_manager.update_bot(self.bot.get_id, {Bot.BOT_STATUS: Bot.STATUS_ERROR}) break print( strftime('%Y%m%d%H%M%S') + ' finished arbitrage process for bot#' + str(self.bot.get_id())) self.logger.info("#" + str(self.bot.get_id()) + ": Finished.") # TODO: uncomment this line # self.bot_manager.update_bot(self.bot.get_id, {Bot.BOT_STATUS: Bot.STATUS_FINISHED}) def init_exchanges(self): """ Find the exchanges that are needed for the arbitrage """ exchange_one = self.fetch_exchanges(self.arbitrage.get_exchange_one()) exchange_two = self.fetch_exchanges(self.arbitrage.get_exchange_two()) if exchange_one and exchange_two: self.exchanges_one = exchange_one self.exchanges_two = exchange_two else: raise Exception("Exchanges not found.") def fetch_exchanges(self, exchange): """ Fetch exchanges :param exchange: :return: """ if exchange == "all": return self.exchange_manager.get_exchanges( {Exchange.EXCHANGE_USER: self.bot.get_userid()}) else: return self.exchange_manager.get_exchanges({ Exchange.EXCHANGE_USER: self.bot.get_userid(), Exchange.EXCHANGE_NAME: exchange }) def check_balance(self): # TODO: might place this somewehere else for more process to acces? maybe an abstract class that each process can inherent """ Check balances on all exchanges :return bool: """ exchanges = self.exchanges_one.copy() exchanges.extend(self.exchanges_two) for exchange in exchanges: balance = self.exchange_manager.fetch_balance( exchange, self.bot.get_pair()) if not balance or float(balance) < float(self.BALANCE_LIMIT): raise Exception("Not enough balance on exchange '" + exchange.get_name() + "' / '" + balance + "'.") return True def do_magic(self): """ Take exchange as base exchange (A) value, loop through the other exchange (B) and compare value with base exchange, if base exchange is value A significantly lower than B, put base exchange A as BUY and exchange B as SELL. """ # TODO: check on fees, make a real arbitrage transaction and see what fees are applied print("magic starts") amount = float(self.bot.get_amount()) win_limit = float(self.bot.get_win_limit()) pair = self.bot.get_pair() buy_and_sell = {} for a_exchange in self.exchanges_one: a_fee_percentage = self.exchange_manager.get_exchange_trading_fee( a_exchange, pair, "maker") # maker costs in % a_price = self.exchange_manager.get_market_price( a_exchange, pair, "buy") # get price from base exchange fee_maker = ( a_price * amount ) * a_fee_percentage # calculate the maker fee in 'base currency BTC/USD (BTC)?' for b_exchange in self.exchanges_two: # now loop through the remainder of the exchanges if b_exchange != a_exchange: # TODO: INTERCHANGE THESE 2 IF's # if b_exchange.get_id() != a_exchange.get_id(): b_fee_percentage = self.exchange_manager.get_exchange_trading_fee( b_exchange, pair, "taker") # taker costs in %t b_price = self.exchange_manager.get_market_price( b_exchange, pair, "sell") # get price from comparison exchange fee_taker = ( b_price * amount ) * b_fee_percentage # calculate the taker fee in #? turnover = (b_price - a_price) * amount # calculate the turnover total_fees = fee_maker + fee_taker # calculate the total fees min_profit = (a_price * amount) * ( win_limit / float(100) ) # calculate the required profit if (turnover - total_fees) > min_profit: # if these exchanges make the required profit, set the list up to place the orders buy_and_sell["buy"] = { "exchange": a_exchange, "price": a_price } buy_and_sell["sell"] = { "exchange": b_exchange, "price": b_price } return buy_and_sell def place_orders(self, verdict): """ Place an order depending on the verdict. Buy from exchange1 and sell to exchange 2 and vice versa. :param verdict: :return: """ print(verdict) print("placing order") # first do the buy order buy_order = verdict["buy"] buy_result = self.exchange_manager.place_order(buy_order["exchange"], self.bot.get_pair(), "buy", self.bot.get_amount()) #TODO: CREATE ORDER MODEL WITH BUY TYPE if not buy_result["id"]: raise Exception("Could not place BUY order. " + buy_result["info"]) # then if it went sucessfully place the sell order sell_order = verdict["sell"] sell_result = self.exchange_manager.place_order( sell_order["exchange"], self.bot.get_pair(), "sell", self.bot.get_amount()) #TODO: CREATE ORDER MODEL WITH SELL TYPE if not sell_result["id"]: raise Exception("Could not place SELL order. " + sell_result["info"]) self.open_order = True def finish_bot(self): """ Set local open order variable to true so no new order will be placed for this bot :return: """ #TODO: fetch the order data wait untill closed and save them in order DB? self.bot.set_status(Bot.STATUS_FINISHED)
def worker(self, ip): """ :param ip: str IPv4 address to use in the request to update """ Logger.info(f"using IP {ip}")
def __init__(self, config): Logger.info(config)