def get_thc_data(row): data = [[ ' Health center level', 'HCs', 'Vector control', 'Diagnosis', 'Treatment', 'Severe case mngt.', 'IRT', 'M&E', 'ICCM', 'Supply Management', 'SBCC' ]] woreda_hc = Dataset.merged_woreda_hc() for index, r in woreda_hc[woreda_hc['woreda'] == row['woreda']].iterrows(): data.append([ '', Utils.short(r['hfname']), Utils.n(r['vect_y']), Utils.n(r['dx_y']), Utils.n(r['rx_y']), Utils.n(r['mxsevr']), Utils.n(r['IRT']), Utils.n(r['sme_y']), Utils.n(r['iccm_y']), Utils.n(r['scm_y']), Utils.n(r['sbcc_y']) ]) return data
def execute(self, response): metas = response.meta try: jsonValue = json.loads(response.body) print response.url print response.status print len(jsonValue) print "---" if 'erro' in jsonValue: print jsonValue else: countSaved = 0 for item in jsonValue: value = int(Utils.toUTF8(item['Value'])) label = Utils.toUTF8(item['Label']) countSaved = countSaved + 1; print self._model.add(value, metas['codigoTipoVeiculo'], metas['codigoTabelaReferencia'], label) if countSaved > 0: tabelaReferencia = TabelaReferenciaModel(self._db) print tabelaReferencia.setCrawled(metas['codigoTipoVeiculo'], metas['codigoTabelaReferencia']) except e: print e # enddef
def execute(self, cmd, params=()): result = None try: newParams = () for item in params: newParams = newParams + (self.__conn.escape_string(item) if isinstance(item, str) else item, ) cur = self.__conn.cursor() cur.execute(cmd % newParams) self.__conn.commit() result = Utils.structReturn( False, { 'total': cur.rowcount if hasattr(cur, 'rowcount') else 0, 'id': self.__conn.lastrowid if hasattr(self.__conn, 'lastrowid') else 0 }) except MySQLdb.Error, e: result = Utils.structReturn( True, "Error %d: %s" % (e.args[0], e.args[1]))
def is_stock_codes_update_needed(): update_dt = Utils.r.get('stock_codes_update_time') if not update_dt: return True # timezone = 'Asia/Shanghai' update_dt = datetime.strptime(update_dt, "%Y-%m-%d %H:%M:%S") now = datetime.now() if Utils.is_trading_day(now): compare_dt = now.replace(hour=9, minute=0, second=0, microsecond=0) if now > compare_dt and update_dt < compare_dt: return True else: return False else: last_trading_day = Utils.previous_trading_day(now) compare_dt = last_trading_day.replace(hour=9, minute=0, second=0, microsecond=0) if update_dt <= compare_dt: return True else: return False
def select(self, cmd, params = ()): result = None cur = None try: newParams = () for item in params: newParams = newParams + (self.__conn.escape_string(item) if isinstance(item, str) else item,) cur = self.__conn.cursor() cur.execute(cmd % newParams) number_rows = cur.rowcount number_columns = len(cur.description) name_columns = [column[0] for column in cur.description] if number_rows >= 1 and number_columns > 1: result = [dict(zip(name_columns, item)) for item in cur.fetchall()] else: result = [dict(zip(name_columns, item[0])) for item in cur.fetchall()] result = {'total': number_rows, 'itens': result} result = Utils.structReturn(False, result) except MySQLdb.Error, e: result = Utils.structReturn(True, "Error %d: %s" % (e.args[0],e.args[1]))
def select(self, cmd, params=()): result = None cur = None try: newParams = () for item in params: newParams = newParams + (self.__conn.escape_string(item) if isinstance(item, str) else item, ) cur = self.__conn.cursor() cur.execute(cmd % newParams) number_rows = cur.rowcount number_columns = len(cur.description) name_columns = [column[0] for column in cur.description] if number_rows >= 1 and number_columns > 1: result = [ dict(zip(name_columns, item)) for item in cur.fetchall() ] else: result = [ dict(zip(name_columns, item[0])) for item in cur.fetchall() ] result = {'total': number_rows, 'itens': result} result = Utils.structReturn(False, result) except MySQLdb.Error, e: result = Utils.structReturn( True, "Error %d: %s" % (e.args[0], e.args[1]))
def __listAllBase(self, showDeleted, page=1, tpp=50, search=''): params = (','.join(self._selectItens), self._table, ('1=1' if showDeleted else 'ativo=1'), search) sql = "SELECT %s FROM %s WHERE %s %s" result = self._db.select(sql, params) if result['err'] is True: self.__setError(result['ret']) return Utils.structReturn(True, 0) else: total = result['ret']['total'] totalpag = 0 if page > 0: ini = (page * tpp) - tpp if ini <= 0: ini = 0 sql += " LIMIT %d, %d " params = params + (ini, tpp) result = self._db.select(sql, params) totalpag = result['ret']['total'] ## endif if (total and not page) or (totalpag and page): return result else: self.__setError("Nenhum dado encontrado!") return Utils.structReturn(True, 0)
def __init__(self): self.console = Console() self.location = Location() self.utils = Utils() self.home = str(Path.home()) self.version_manager_loc = "sh $HOME/.config/zsh-config/groups/Version-Manager/"
def generate_hp_bar_chart(woreda_hp): # shorten the name of each health post categories = woreda_hp['name_hp'].apply(lambda x: Utils.short(x)) data = woreda_hp['mb_2008_y'] return Utils.generate_bar_chart(categories, data, ylabel="# of malaria cases")
class VersionManager: def __init__(self): self.console = Console() self.location = Location() self.utils = Utils() self.home = str(Path.home()) self.version_manager_loc = "sh $HOME/.config/zsh-config/groups/Version-Manager/" def pyenv(self): self.console.print() self.console.print("Installing [bold red]pyenv[/]...") os.system(self.version_manager_loc + "pyenv.sh") def rbenv(self): self.console.print() self.console.print("Installing [bold red]rbenv[/]...") os.system(self.version_manager_loc + "rbenv.sh") def sdkman(self): self.console.print() self.console.print("Installing [bold red]SDK Man[/]...") os.system(self.version_manager_loc + "sdkman.sh") def gvm(self): self.console.print() self.console.print("Installing [bold red]gvm[/]...") os.system(self.version_manager_loc + "gvm.sh") def run(self): tasks = { 0: "All", 1: "pyenv", 2: "rbenv", 3: "sdkman", 4: "gvm", } self.console.rule("Install package from?") self.utils.tasks_print(tasks) choice = self.console.input("What is your [bold red]choice[/]? :smiley: ") self.console.print(f'You entered {choice}') choice = int(choice) if choice == 0: self.pyenv() self.rbenv() self.sdkman() elif choice == 1: self.pyenv() elif choice == 2: self.rbenv() elif choice == 3: self.sdkman() elif choice == 4: self.gvm()
def generate_hew_bar_chart(row): woreda_hp = Dataset.merged_woreda_hp() woreda_hp = woreda_hp[woreda_hp['woreda'] == row['woreda']] sorted_data = woreda_hp.sort_values(by=['total_extension']) category = sorted_data['hpname'].apply(lambda x: Utils.short(str(x))) data = sorted_data['total_extension'] return Utils.generate_bar_chart(category, data, ylabel='# of HEWs')
def generate_pie_chart(row): woreda_hp = Dataset.merged_woreda_hp_mb() categories, data = Utils.rearrange( woreda_hp[(woreda_hp['woreda'] == row['woreda']) & (woreda_hp['mb_2008_y'] > 0)], 'name_hp', 'mb_2008_y') # shorten the name of each health post categories = categories.apply(lambda x: Utils.short(x)) return Utils.generate_pie_chart(categories, data)
def get_data(row): data = [[ Utils.p('IV. Finance', 10, True, '#0085B7', align=Align.LEFT), '', '', '', '' ], [ Utils.p('Woreda', 8, True, align=Align.LEFT), '', '', Utils.p('Health center', 8, True, align=Align.LEFT), '' ], [ Utils.p('Woreda allocate funds for malaria activities', 7, align=Align.LEFT), Utils.p(row['bgt'], 7, align=Align.LEFT), '', '', Utils.p('Participate in health insurance', 7, True, '#5A5A5C') ]] woreda_hc = Dataset.merged_woreda_hc() for index, r in woreda_hc[woreda_hc['woreda'] == row['woreda']].iterrows(): data.append([ '', '', '', Utils.p(Utils.short(r['hfname']), 7, align=Align.LEFT), Utils.p('Yes' if r['insu'] == 1 else 'No', 7) ]) return data
def get_dx_hc_data(row): data = [[ Utils.p('Malaria diagnosis method used by HCs', 8, True, align=Align.LEFT), '', '', '' ], [ '', Utils.p('Light Microscopy', 7, True, '#5A5A5C'), Utils.p('RDT', 7, True, '#5A5A5C'), Utils.p('Both', 7, True, '#5A5A5C') ]] woreda_hc = Dataset.merged_woreda_hc() for index, r in woreda_hc[woreda_hc['woreda'] == row['woreda']].iterrows(): data.append([ Utils.p(Utils.short(r['hfname']), 7, align=Align.LEFT), Utils.p(r['dxmic'], 7), Utils.p(r['dxrdt'], 7), Utils.p(r['dxboth'], 7) ]) return data
def generate_bar_chart(row): woreda_hc = Dataset.merged_woreda_hc_mb() woreda_hc = woreda_hc[woreda_hc['woreda'] == row['woreda']] sorted_data = woreda_hc.sort_values(by=['mb_2008_y']) category = sorted_data['name_hc'].apply(lambda x: Utils.short(x)) data = sorted_data['mb_2008_y'] return Utils.generate_bar_chart(category, data, ylabel='# of malaria cases')
def get_hew_tree_map(row): woreda_hp = Dataset.merged_woreda_hp() sizes = [] labels = [] for index, r in woreda_hp[(woreda_hp['woreda'] == row['woreda']) & ( woreda_hp['total_extension'] > 0)].iterrows(): sizes.append(r['total_extension']) labels.append('{}\n{:.0f}'.format(Utils.short(r['hpname']), r['total_extension'])) return Utils.generate_tree_map(labels, sizes)
def prepare(self): buffers = self.buffers scope = self.scope sub_group_count = self.sub_group_count self.codes = np.frombuffer(buffers['codes'], dtype='<U6') self.names = np.frombuffer(buffers['names'], dtype='<U4') # 快照 数据 msl_interval = 5 msl_check_points = Utils.get_check_points(msl_interval) msl_cols = [ 'open', 'close', 'now', 'high', 'low', 'turnover', 'volume' ] self.msl = D3Array(msl_check_points, self.codes, msl_cols, msl_interval, buffer=buffers['msl']) # market snapshots # 分时 数据 fsl_interval = 60 fsl_check_points = Utils.get_check_points(fsl_interval) fsl_cols = ['price', 'volume'] self.fsl = D3Array(fsl_check_points, self.codes, fsl_cols, fsl_interval, buffer=buffers['fsl']) # 分时 图 # 统计 数据 stat_cols = [ 'zhangfu', 'junjia', 'liangbi', 'zhangsu', 'tingban', 'fsto' ] self.stat = D3Array(fsl_check_points, self.codes, stat_cols, fsl_interval, buffer=buffers['stat']) # 分时 统计 # 基本 信息 basics_cols = ['zt_price', 'dt_prics', 'ma5vpm', 'ltgb'] self.basics = D2Array(self.codes, basics_cols, buffer=buffers['basics']) if scope is None: self.sina = Sina(self.codes, group_count=sub_group_count) else: self.sina = Sina(self.codes[scope[0]:scope[1]], group_count=sub_group_count)
def get_data(row): return [ ['', '', GeneralInfo.generate_map(row) ], # Utils.p('Contact person: {}'.format('+251...'), 7, True) [ Utils.p('I. General Information', 10, True, '#0085B7', align=Align.LEFT), '', '' ], [ Utils.p('Number of Health Facilities', 8, True, align=Align.LEFT), '', '' ], [ Utils.p('Health Centers', align=Align.LEFT), Utils.p(row['num_HC']), '' ], [ Utils.p('Health Posts', align=Align.LEFT), Utils.p(row['num_HP']), '' ], [ Utils.p('Population - 2008 EC', align=Align.LEFT), Utils.p(row['pop2008']), '' ] ]
def get_summary_data(row): woreda_hc = Dataset.merged_woreda_hc_mb() woreda_hc = woreda_hc[woreda_hc['woreda'] == row['woreda']].sum() woreda_hp = Dataset.merged_woreda_hp_mb() woreda_hp = woreda_hp[woreda_hp['woreda'] == row['woreda']].sum() return [[ Utils.p('II. Baseline Malaria Burden, 2008 EFY', 10, True, '#0085B7', align=Align.LEFT), '', '', '' ], [ Utils.p('Malaria cases reported by', 8, True, align=Align.LEFT), Utils.p('Health Centers', 7, True, '#5A5A5C'), Utils.p('Health Posts', 7, True, '#5A5A5C'), Utils.p('Total', 7, True, '#5A5A5C') ], [ '', Utils.p(woreda_hc['mb_2008_y']), Utils.p(woreda_hp['mb_2008_y']), Utils.p(woreda_hc['mb_2008_y'] + woreda_hp['mb_2008_y']) ]]
def header_footer(canvas, doc): # Save the state of our canvas so we can draw on it canvas.saveState() # header canvas.drawImage('assets/images/FMOH.png', doc.leftMargin + 10, doc.height + doc.topMargin, 48, 48) canvas.drawImage('assets/images/EPHI.png', doc.leftMargin + doc.width - 53, doc.height + doc.topMargin, 48, 48) header = Utils.p('Malaria Elimination Baseline', 18, True, '#0085B7') w, h = header.wrap(doc.width, doc.topMargin) header.drawOn(canvas, doc.leftMargin, doc.height + doc.topMargin + h + 20) header = Utils.p('Woreda Factsheet', 16, False, '#0085B7') w, h = header.wrap(doc.width, doc.topMargin) header.drawOn(canvas, doc.leftMargin, doc.height + doc.topMargin + h) canvas.setLineWidth(.3) #canvas.setStrokeColorRGB(0, 1, 0.3) canvas.line(doc.leftMargin + 5.5, doc.bottomMargin - 15, doc.width + 14, doc.bottomMargin - 15) # Footer canvas.drawImage('assets/images/SMMES.png', doc.leftMargin + 5, doc.bottomMargin - 45, 158, 25) canvas.drawImage('assets/images/WHO.png', doc.leftMargin + 188, doc.bottomMargin - 45, 63, 23) # 188:- 158 + 25 +5 canvas.drawImage('assets/images/CHAI.png', doc.leftMargin + 276, doc.bottomMargin - 45, 43, 23) # 276:- 188 + 63 + 25 canvas.drawImage('assets/images/TU.png', doc.leftMargin + 344, doc.bottomMargin - 45, 63, 23) # 344:- 276 + 43 + 25 canvas.drawImage('assets/images/PATH.png', doc.leftMargin + 432, doc.bottomMargin - 45, 49, 19) # 432:- 344 + 63 + 50 # Page number canvas.setFont("mb", 7) canvas.setFillColor('#0085B7') page_num = canvas.getPageNumber() page_num_text = 'Page {}'.format(page_num) # 524:- 158 + 50 + 43 + 50 + 63 + 50 + 110 canvas.drawRightString(doc.leftMargin + 524, doc.bottomMargin - 40, page_num_text) # Release the canvas canvas.restoreState()
class Latest: def __init__(self): self.utils = Utils() self.command = Command() self.console = Console() self.location = Location() def run(self): home = str(Path.home()) stream = self.utils.load_yaml(home + self.location.GITHUB + self.location.REPOSITORIES_FILE) for repo in stream: print() code = repo.get(self.command.CODE) self.console.print("Processing [bold red]" + code + "[/]...") r = requests.get('https://api.github.com/repos/' + code + "/releases/latest") response = r.json() self.console.print("[green]URL[/]: [bold magenta]" + response["html_url"] + "[/]") self.console.print("[green]Tag[/]: [bold magenta]" + response["tag_name"] + "[/]") self.console.print("[green]Publish date[/]: [bold magenta]" + response["published_at"] + "[/]") self.console.print("[green]Release name[/]: [bold magenta]" + response["name"] + "[/]")
def __init__(self): self.config = self.read_config() self.utils = Utils(self.config) self.pi_utils = PiUtils(self.config, self.utils) self.eve = evelink.eve.EVE() self.db = Connect(self.config['database']['data']) self.classes = self.db.base.classes parser = argparse.ArgumentParser() parser.add_argument('--create-db', help='create the databases', action='store_true') parser.add_argument('--pi', help='update the Planetary Interaction cache', action='store_true') parser.add_argument('--losses', help='update the items and ships that have been destroyed', action='store', type=int) parser.add_argument('--alliances', help='update the alliances', action='store_true') self.args = parser.parse_args() if self.args.create_db: self.create_databases() if self.args.pi: self.update_pi() if self.args.losses: self.update_losses() if self.args.alliances: self.update_alliances()
def kline(codes, scale=240, ma=5, length=1023): args = (scale, ma, length) url = 'http://money.finance.sina.com.cn/quotes_service/api/json_v2.php/' \ 'CN_MarketData.getKLineData?symbol={}&scale={}&ma={}&datalen={}' urls = list(map( lambda x: url.format('sh'+x[-6:], *args) if x[0]=='6' else \ url.format('sz'+x[-6:], *args), codes )) # loop = asyncio.get_event_loop() # results = loop.run_until_complete(Utils.fetch_all(urls)) results = Utils.fetch_all(urls) klines = list( map( lambda x: json.loads( re.sub('(\w+)\s?:\s?("?[^",]+"?,?)', "\"\g<1>\":\g<2>", x) ), results)) stocks = {} for idx, code in enumerate(codes): stocks[code] = klines[idx] return stocks
def login_user(): if request.method == 'POST': username = request.form['username'] password = request.form['password'] user = UserModel.find_by_username(username) try: #if user and safe_str_cmp(password, user.password): if user and Utils.check_hashed_password(password, user.password): session['username'] = user.username if user.username in current_app.config.get('ADMIN', '').split(','): session['admin'] = True else: session['admin'] = None return redirect(url_for("webmodels.index")) else: session['username'] = None session['admin'] = None return "Invalid credentials! please try again." except: session['username'] = None session['admin'] = None return "Invalid credentials! please try again." return render_template("users/login.html")
def post(self): json_data = request.get_json() if not json_data: return {'message': 'No input data provided'} user_data = user_schema.load(json_data) if not UserModel.find_by_username(username=user_data['username']): return {'message': 'Username is not exists'} user_ = user_schema.dump(UserModel.find_by_username(username=user_data['username'])) if Utils.decrypt_pass(user_data['password'], user_['password']): access_token = create_access_token(user_['username']) refresh_token = create_refresh_token(user_['username']) return { 'message': "Login successful!", 'access_token': access_token, 'refresh_token': refresh_token } else: return { 'message': 'Username or password wrong!' }
def real(self, codes, group_count=None, array=None): length = len(codes) codes_with_prefix = list( map(lambda x: 'sh' + x[-6:] if x[0] == '6' else 'sz' + x[-6:], codes)) if group_count is None: group_size = self.max_group_size group_count = math.ceil(length / group_size) else: min_group_count = math.ceil(length / self.max_group_size) if group_count < min_group_count: group_count = min_group_count group_size = math.ceil(length / group_count) code_lists = list([ ','.join(codes_with_prefix[idx * group_size:min((idx + 1) * group_size, length)]) for idx in range(self.group_count) ]) urls = [self.stock_api + stock_list for stock_list in code_lists] # loop = asyncio.get_event_loop() # result = loop.run_until_complete(Utils.fetch_all(urls)) result = Utils.fetch_all(urls) return self.parse_real_data(''.join(result), len(self.codes), array=array)
def set_basics(self): # set zt_price returns = self.send_processes_message(self.hq_pipes, { 'type': 'function', 'function': 'check', 'args': [], 'kwargs': {} }) idx = returns[0]['msl'] self.basics.data[:, 0] = np.around(self.msl.data[idx, :, 1] * 1.1, decimals=2) self.basics.data[:, 1] = np.around(self.msl.data[idx, :, 1] * 0.9, decimals=2) # set ma5vpm self.calculate_ma5vpm() # set ltgb dt = Utils.previous_trading_day(datetime.now()) df = TradingDay.ts.daily_basic(trade_date=dt.strftime('%Y%m%d')) for index, code in enumerate(self.codes): scode = code + '.SH' if code[0] == '6' else code + '.SZ' result = df.loc[df['ts_code'] == scode, 'float_share'] if len(result) == 0: continue self.basics.data[index, 3] = result.iloc[0]
def get_lastest_stock_codes(self): old_codes = Utils.get_stock_codes() code_filter = Utils.code_filter step_size = 1000 codes = [] for market in Utils.markets: count = self.tdx.get_security_count(market) print(market, count, end=' : ') steps = math.ceil(count / step_size) total = 0 for step in range(steps): result = self.tdx.get_security_list(market, step_size * step) print(str(step) + '/' + str(steps), end=", ") for item in result: code = item['code'].strip() if code[:3] in code_filter[market]: codes.append(code) total += 1 market_name = '深市' if market == 0 else '沪市' print(market_name + ' A股 总数: ' + str(total)) print('沪深 A股 总数:' + str(len(codes))) return codes
def get(self, id): if not id: return False sql = "SELECT %s FROM %s WHERE id = %d" result = self._db.select( sql, [','.join(self._selectItens), self._table, id]) if result['err'] is True: self.__setError(result['ret']) return Utils.structReturn(True, 0) else: if result['ret']['total'] > 0: return Utils.structReturn(False, result['ret']['itens']) else: self.__setError("Nenhum dado encontrado!") return Utils.structReturn(True, 0)
def execute(self, cmd, params = ()): result = None try: newParams = () for item in params: newParams = newParams + (self.__conn.escape_string(item) if isinstance(item, str) else item,) cur = self.__conn.cursor() cur.execute(cmd % newParams) self.__conn.commit() result = Utils.structReturn(False, {'total': cur.rowcount if hasattr(cur, 'rowcount') else 0, 'id': self.__conn.lastrowid if hasattr(self.__conn,'lastrowid') else 0}) except MySQLdb.Error, e: result = Utils.structReturn(True, "Error %d: %s" % (e.args[0],e.args[1]))
def parseById(self, tipoVeiculo, id): for sel in self._DOMtree.xpath('//select[@id="%s"]/option' % (id)): text = sel.xpath('text()').extract() value = sel.xpath('@value').extract() if type(text).__name__ == 'list': text = text[0] if type(value).__name__ == 'list': value = value[0] value = int(Utils.toUTF8(value)) text = str(Utils.toUTF8(text)) print self._model.add(value, tipoVeiculo, text) # endfor # enddef
def reset_suspended_stock_codes(): now = datetime.now() time1 = now.replace(hour=8, minute=45, second=0, microsecond=0) time2 = now.replace(hour=15, minute=0, second=0, microsecond=0) if time1 < now < time2: print('请在8:30之前 或 15:05之后, 初始化 停牌股票') return suspended_codes = [] market_snapshot = Sina(Utils.get_stock_codes()).market_snapshot() for code in market_snapshot: stock = market_snapshot[code] if stock['turnover'] == 0 and stock['close'] != 0: suspended_codes.append(code) Utils.save_suspended_stock_codes(suspended_codes) return suspended_codes
def market_snapshot(self, array=None): urls = [self.stock_api + stock_list for stock_list in self.code_lists] # loop = asyncio.get_event_loop() # result = loop.run_until_complete(Utils.fetch_all(urls)) result = Utils.fetch_all(urls) return self.parse_real_data(''.join(result), len(self.codes), array=array)
def execute(self, response): metas = response.meta try: jsonValue = json.loads(response.body) print response.url print response.status print len(jsonValue) print "---" if 'erro' in jsonValue: print jsonValue print "Deu erro :(" else: countSaved = 0 CodigoFipe = Utils.toUTF8(jsonValue['CodigoFipe']) Modelo = Utils.toUTF8(jsonValue['Modelo']) Valor = Utils.toUTF8(jsonValue['Valor']) try: Valor = float((re.sub("[^0-9,]+", "", Valor)).replace(",",".")) except e: Valor = 0 if Valor>0: print self._model.add(CodigoFipe, metas['codigoAnoModelo'], metas['codigoModelo'], metas['codigoMarca'], metas['codigoTipoVeiculo'], metas['codigoTabelaReferencia'], Modelo, Valor) countSaved = countSaved + 1 if countSaved > 0: anoModelo = AnoModeloModel(self._db) anoModelo.setCrawled(metas['codigoTipoVeiculo'], metas['codigoTabelaReferencia'], metas['codigoMarca'], metas['codigoModelo'], metas['codigoAnoModelo']) except e: print 'Except =>' print e # enddef
class Command(): def __init__(self): self.config = self.read_config() self.utils = Utils(self.config) self.pi_utils = PiUtils(self.config, self.utils) self.eve = evelink.eve.EVE() self.db = Connect(self.config['database']['data']) self.classes = self.db.base.classes parser = argparse.ArgumentParser() parser.add_argument('--create-db', help='create the databases', action='store_true') parser.add_argument('--pi', help='update the Planetary Interaction cache', action='store_true') parser.add_argument('--losses', help='update the items and ships that have been destroyed', action='store', type=int) parser.add_argument('--alliances', help='update the alliances', action='store_true') self.args = parser.parse_args() if self.args.create_db: self.create_databases() if self.args.pi: self.update_pi() if self.args.losses: self.update_losses() if self.args.alliances: self.update_alliances() def read_config(self, path='config.json'): with open(path) as cfg: config = json.load(cfg) assert(config) return config def create_databases(self): from sql import initialize_sql from sql.pi import Pi from sql.losses import ItemsLost, Kills from sql.alliances import Corporation, Alliance initialize_sql(self.db.engine) def update_pi(self): print('updating PI stats...') for system_name in self.config['statistics']['pi_systems']: system = self.utils.lookup_system(system_name).__dict__ for tier in self.config['statistics']['pi_tiers']: self.pi_utils.store_prices(tier, system['solarSystemID']) def update_losses(self): '''Pull the kills from zKillboard and store them in the database''' print('updating losses...') losses = self.db items_lost = {} alliance_ids = [] alliances_requested = [] time_format = '%Y-%m-%d %H:%M:%S' start_time = (datetime.datetime.now() + relativedelta(months=-1)).strftime('%Y%m%d%H%I') # add alliance ids from coalitions to alliance_ids for coalition in self.config['coalitions']: alliance_ids.extend(self.config['coalitions'][coalition]) for alliance_id in alliance_ids: # what? if alliance_id in alliances_requested: continue alliances_requested.append(alliance_id) kb_url = 'https://zkillboard.com/api/kills/allianceID/%s/startTime/%s' % (alliance_id, start_time) # USE REQUESTS JSON page = requests.get(kb_url) data = json.loads(page.text) print('Working on: %s' % (alliance_id)) # Example #{"characterID":926924867,"characterName":"clavo oxidado", #"corporationID":98390683,"corporationName":"Hogyoku","allianceID":1354830081, #"allianceName":"Goonswarm Federation","factionID":0,"factionName":"","securityStatus":-0.9,"damageDone":0,"finalBlow":0,"weaponTypeID":5439,"shipTypeID":0} # https://zkillboard.com/kill/54204593/ for row in data: kill_time = datetime.datetime.strptime(row['killTime'], time_format) kill_id = row['killID'] # See if the killID already exists in the database query = losses.session.query(losses.base.classes.kills).filter_by(killID=kill_id).first() if query: print('killID already exists, skipping...') continue # The kill itself kill = losses.base.classes.kills(killID=kill_id, shipTypeID=row['victim']['shipTypeID'], killTime=kill_time, characterID=row['victim']['characterID'], corporationID=row['victim']['corporationID'], corporationName=row['victim']['corporationName'], allianceID=row['victim']['allianceID']) # Attach the items lost to the kill for line in row['items']: item = losses.base.classes.items_lost(typeID=line['typeID']) kill.items_lost_collection.append(item) for line in row['attackers']: attacker = losses.base.classes.attacker(weaponTypeID=line['weaponTypeID'], allianceID=line['allianceID'], corporationName=line['corporationName'], shipTypeID=line['shipTypeID'], characterName=line['characterName'], characterID=line['characterID'], allianceName=line['allianceName']) kill.attacker_collection.append(attacker) losses.session.add(kill) losses.session.commit() def update_alliances(self): #{'id': 99000304, 'member_count': 8, 'ticker': 'PIGS', 'name': 'THE SPACE P0LICE', 'executor_id': 98038175, 'member_corps': {699225816: {'id': 699225816, 'timestamp': 1423864860}, 875152542 print('Deleting alliances and corporations...') self.db.session.query(self.db.base.classes.alliances).delete() self.db.session.query(self.db.base.classes.corporations).delete() self.db.session.commit() print('Adding current alliances...') alliances = self.eve.alliances().result for id_ in alliances.keys(): member_count = alliances[id_]['member_count'] ticker = alliances[id_]['ticker'] name = alliances[id_]['name'] executor_id = alliances[id_]['executor_id'] member_corps = alliances[id_]['member_corps'] alliance = self.db.base.classes.alliances(id=id_, member_count=member_count, ticker=ticker, name=name, executor_id=executor_id) for c in member_corps.keys(): print(member_corps[c]) corp = self.db.base.classes.corporations(alliance_id=member_corps[c]['id']) alliance.corporations_collection.append(corp) self.db.session.add(alliance) self.db.session.commit()