def get_thc_data(row):
data = [[
' Health center level', 'HCs', 'Vector control', 'Diagnosis',
'Treatment', 'Severe case mngt.', 'IRT', 'M&E', 'ICCM',
'Supply Management', 'SBCC'
]]
woreda_hc = Dataset.merged_woreda_hc()
for index, r in woreda_hc[woreda_hc['woreda'] ==
row['woreda']].iterrows():
data.append([
'',
Utils.short(r['hfname']),
Utils.n(r['vect_y']),
Utils.n(r['dx_y']),
Utils.n(r['rx_y']),
Utils.n(r['mxsevr']),
Utils.n(r['IRT']),
Utils.n(r['sme_y']),
Utils.n(r['iccm_y']),
Utils.n(r['scm_y']),
Utils.n(r['sbcc_y'])
])
return data
def execute(self, response): metas = response.meta try: jsonValue = json.loads(response.body) print response.url print response.status print len(jsonValue) print "---" if 'erro' in jsonValue: print jsonValue else: countSaved = 0 for item in jsonValue: value = int(Utils.toUTF8(item['Value'])) label = Utils.toUTF8(item['Label']) countSaved = countSaved + 1; print self._model.add(value, metas['codigoTipoVeiculo'], metas['codigoTabelaReferencia'], label) if countSaved > 0: tabelaReferencia = TabelaReferenciaModel(self._db) print tabelaReferencia.setCrawled(metas['codigoTipoVeiculo'], metas['codigoTabelaReferencia']) except e: print e # enddef
def execute(self, cmd, params=()):
result = None
try:
newParams = ()
for item in params:
newParams = newParams + (self.__conn.escape_string(item)
if isinstance(item, str) else item, )
cur = self.__conn.cursor()
cur.execute(cmd % newParams)
self.__conn.commit()
result = Utils.structReturn(
False, {
'total':
cur.rowcount if hasattr(cur, 'rowcount') else 0,
'id':
self.__conn.lastrowid
if hasattr(self.__conn, 'lastrowid') else 0
})
except MySQLdb.Error, e:
result = Utils.structReturn(
True, "Error %d: %s" % (e.args[0], e.args[1]))
def is_stock_codes_update_needed():
update_dt = Utils.r.get('stock_codes_update_time')
if not update_dt:
return True
# timezone = 'Asia/Shanghai'
update_dt = datetime.strptime(update_dt, "%Y-%m-%d %H:%M:%S")
now = datetime.now()
if Utils.is_trading_day(now):
compare_dt = now.replace(hour=9, minute=0, second=0, microsecond=0)
if now > compare_dt and update_dt < compare_dt:
return True
else:
return False
else:
last_trading_day = Utils.previous_trading_day(now)
compare_dt = last_trading_day.replace(hour=9,
minute=0,
second=0,
microsecond=0)
if update_dt <= compare_dt:
return True
else:
return False
def select(self, cmd, params = ()):
result = None
cur = None
try:
newParams = ()
for item in params:
newParams = newParams + (self.__conn.escape_string(item) if isinstance(item, str) else item,)
cur = self.__conn.cursor()
cur.execute(cmd % newParams)
number_rows = cur.rowcount
number_columns = len(cur.description)
name_columns = [column[0] for column in cur.description]
if number_rows >= 1 and number_columns > 1:
result = [dict(zip(name_columns, item)) for item in cur.fetchall()]
else:
result = [dict(zip(name_columns, item[0])) for item in cur.fetchall()]
result = {'total': number_rows, 'itens': result}
result = Utils.structReturn(False, result)
except MySQLdb.Error, e:
result = Utils.structReturn(True, "Error %d: %s" % (e.args[0],e.args[1]))
def select(self, cmd, params=()):
result = None
cur = None
try:
newParams = ()
for item in params:
newParams = newParams + (self.__conn.escape_string(item)
if isinstance(item, str) else item, )
cur = self.__conn.cursor()
cur.execute(cmd % newParams)
number_rows = cur.rowcount
number_columns = len(cur.description)
name_columns = [column[0] for column in cur.description]
if number_rows >= 1 and number_columns > 1:
result = [
dict(zip(name_columns, item)) for item in cur.fetchall()
]
else:
result = [
dict(zip(name_columns, item[0]))
for item in cur.fetchall()
]
result = {'total': number_rows, 'itens': result}
result = Utils.structReturn(False, result)
except MySQLdb.Error, e:
result = Utils.structReturn(
True, "Error %d: %s" % (e.args[0], e.args[1]))
def __listAllBase(self, showDeleted, page=1, tpp=50, search=''):
params = (','.join(self._selectItens), self._table,
('1=1' if showDeleted else 'ativo=1'), search)
sql = "SELECT %s FROM %s WHERE %s %s"
result = self._db.select(sql, params)
if result['err'] is True:
self.__setError(result['ret'])
return Utils.structReturn(True, 0)
else:
total = result['ret']['total']
totalpag = 0
if page > 0:
ini = (page * tpp) - tpp
if ini <= 0:
ini = 0
sql += " LIMIT %d, %d "
params = params + (ini, tpp)
result = self._db.select(sql, params)
totalpag = result['ret']['total']
## endif
if (total and not page) or (totalpag and page):
return result
else:
self.__setError("Nenhum dado encontrado!")
return Utils.structReturn(True, 0)
def __init__(self):
self.console = Console()
self.location = Location()
self.utils = Utils()
self.home = str(Path.home())
self.version_manager_loc = "sh $HOME/.config/zsh-config/groups/Version-Manager/"
def generate_hp_bar_chart(woreda_hp):
# shorten the name of each health post
categories = woreda_hp['name_hp'].apply(lambda x: Utils.short(x))
data = woreda_hp['mb_2008_y']
return Utils.generate_bar_chart(categories,
data,
ylabel="# of malaria cases")
class VersionManager:
def __init__(self):
self.console = Console()
self.location = Location()
self.utils = Utils()
self.home = str(Path.home())
self.version_manager_loc = "sh $HOME/.config/zsh-config/groups/Version-Manager/"
def pyenv(self):
self.console.print()
self.console.print("Installing [bold red]pyenv[/]...")
os.system(self.version_manager_loc + "pyenv.sh")
def rbenv(self):
self.console.print()
self.console.print("Installing [bold red]rbenv[/]...")
os.system(self.version_manager_loc + "rbenv.sh")
def sdkman(self):
self.console.print()
self.console.print("Installing [bold red]SDK Man[/]...")
os.system(self.version_manager_loc + "sdkman.sh")
def gvm(self):
self.console.print()
self.console.print("Installing [bold red]gvm[/]...")
os.system(self.version_manager_loc + "gvm.sh")
def run(self):
tasks = {
0: "All",
1: "pyenv",
2: "rbenv",
3: "sdkman",
4: "gvm",
}
self.console.rule("Install package from?")
self.utils.tasks_print(tasks)
choice = self.console.input("What is your [bold red]choice[/]? :smiley: ")
self.console.print(f'You entered {choice}')
choice = int(choice)
if choice == 0:
self.pyenv()
self.rbenv()
self.sdkman()
elif choice == 1:
self.pyenv()
elif choice == 2:
self.rbenv()
elif choice == 3:
self.sdkman()
elif choice == 4:
self.gvm()
def generate_hew_bar_chart(row):
woreda_hp = Dataset.merged_woreda_hp()
woreda_hp = woreda_hp[woreda_hp['woreda'] == row['woreda']]
sorted_data = woreda_hp.sort_values(by=['total_extension'])
category = sorted_data['hpname'].apply(lambda x: Utils.short(str(x)))
data = sorted_data['total_extension']
return Utils.generate_bar_chart(category, data, ylabel='# of HEWs')
def generate_pie_chart(row):
woreda_hp = Dataset.merged_woreda_hp_mb()
categories, data = Utils.rearrange(
woreda_hp[(woreda_hp['woreda'] == row['woreda'])
& (woreda_hp['mb_2008_y'] > 0)], 'name_hp', 'mb_2008_y')
# shorten the name of each health post
categories = categories.apply(lambda x: Utils.short(x))
return Utils.generate_pie_chart(categories, data)
def get_data(row):
data = [[
Utils.p('IV. Finance', 10, True, '#0085B7', align=Align.LEFT), '',
'', '', ''
],
[
Utils.p('Woreda', 8, True, align=Align.LEFT), '', '',
Utils.p('Health center', 8, True, align=Align.LEFT), ''
],
[
Utils.p('Woreda allocate funds for malaria activities',
7,
align=Align.LEFT),
Utils.p(row['bgt'], 7, align=Align.LEFT), '', '',
Utils.p('Participate in health insurance', 7, True,
'#5A5A5C')
]]
woreda_hc = Dataset.merged_woreda_hc()
for index, r in woreda_hc[woreda_hc['woreda'] ==
row['woreda']].iterrows():
data.append([
'', '', '',
Utils.p(Utils.short(r['hfname']), 7, align=Align.LEFT),
Utils.p('Yes' if r['insu'] == 1 else 'No', 7)
])
return data
def get_dx_hc_data(row):
data = [[
Utils.p('Malaria diagnosis method used by HCs',
8,
True,
align=Align.LEFT), '', '', ''
],
[
'',
Utils.p('Light Microscopy', 7, True, '#5A5A5C'),
Utils.p('RDT', 7, True, '#5A5A5C'),
Utils.p('Both', 7, True, '#5A5A5C')
]]
woreda_hc = Dataset.merged_woreda_hc()
for index, r in woreda_hc[woreda_hc['woreda'] ==
row['woreda']].iterrows():
data.append([
Utils.p(Utils.short(r['hfname']), 7, align=Align.LEFT),
Utils.p(r['dxmic'], 7),
Utils.p(r['dxrdt'], 7),
Utils.p(r['dxboth'], 7)
])
return data
def generate_bar_chart(row):
woreda_hc = Dataset.merged_woreda_hc_mb()
woreda_hc = woreda_hc[woreda_hc['woreda'] == row['woreda']]
sorted_data = woreda_hc.sort_values(by=['mb_2008_y'])
category = sorted_data['name_hc'].apply(lambda x: Utils.short(x))
data = sorted_data['mb_2008_y']
return Utils.generate_bar_chart(category,
data,
ylabel='# of malaria cases')
def get_hew_tree_map(row):
woreda_hp = Dataset.merged_woreda_hp()
sizes = []
labels = []
for index, r in woreda_hp[(woreda_hp['woreda'] == row['woreda']) & (
woreda_hp['total_extension'] > 0)].iterrows():
sizes.append(r['total_extension'])
labels.append('{}\n{:.0f}'.format(Utils.short(r['hpname']),
r['total_extension']))
return Utils.generate_tree_map(labels, sizes)
def prepare(self):
buffers = self.buffers
scope = self.scope
sub_group_count = self.sub_group_count
self.codes = np.frombuffer(buffers['codes'], dtype='<U6')
self.names = np.frombuffer(buffers['names'], dtype='<U4')
# 快照 数据
msl_interval = 5
msl_check_points = Utils.get_check_points(msl_interval)
msl_cols = [
'open', 'close', 'now', 'high', 'low', 'turnover', 'volume'
]
self.msl = D3Array(msl_check_points,
self.codes,
msl_cols,
msl_interval,
buffer=buffers['msl']) # market snapshots
# 分时 数据
fsl_interval = 60
fsl_check_points = Utils.get_check_points(fsl_interval)
fsl_cols = ['price', 'volume']
self.fsl = D3Array(fsl_check_points,
self.codes,
fsl_cols,
fsl_interval,
buffer=buffers['fsl']) # 分时 图
# 统计 数据
stat_cols = [
'zhangfu', 'junjia', 'liangbi', 'zhangsu', 'tingban', 'fsto'
]
self.stat = D3Array(fsl_check_points,
self.codes,
stat_cols,
fsl_interval,
buffer=buffers['stat']) # 分时 统计
# 基本 信息
basics_cols = ['zt_price', 'dt_prics', 'ma5vpm', 'ltgb']
self.basics = D2Array(self.codes,
basics_cols,
buffer=buffers['basics'])
if scope is None:
self.sina = Sina(self.codes, group_count=sub_group_count)
else:
self.sina = Sina(self.codes[scope[0]:scope[1]],
group_count=sub_group_count)
def get_data(row):
return [
['', '', GeneralInfo.generate_map(row)
], # Utils.p('Contact person: {}'.format('+251...'), 7, True)
[
Utils.p('I. General Information',
10,
True,
'#0085B7',
align=Align.LEFT), '', ''
],
[
Utils.p('Number of Health Facilities',
8,
True,
align=Align.LEFT), '', ''
],
[
Utils.p('Health Centers', align=Align.LEFT),
Utils.p(row['num_HC']), ''
],
[
Utils.p('Health Posts', align=Align.LEFT),
Utils.p(row['num_HP']), ''
],
[
Utils.p('Population - 2008 EC', align=Align.LEFT),
Utils.p(row['pop2008']), ''
]
]
def get_summary_data(row):
woreda_hc = Dataset.merged_woreda_hc_mb()
woreda_hc = woreda_hc[woreda_hc['woreda'] == row['woreda']].sum()
woreda_hp = Dataset.merged_woreda_hp_mb()
woreda_hp = woreda_hp[woreda_hp['woreda'] == row['woreda']].sum()
return [[
Utils.p('II. Baseline Malaria Burden, 2008 EFY',
10,
True,
'#0085B7',
align=Align.LEFT), '', '', ''
],
[
Utils.p('Malaria cases reported by',
8,
True,
align=Align.LEFT),
Utils.p('Health Centers', 7, True, '#5A5A5C'),
Utils.p('Health Posts', 7, True, '#5A5A5C'),
Utils.p('Total', 7, True, '#5A5A5C')
],
[
'',
Utils.p(woreda_hc['mb_2008_y']),
Utils.p(woreda_hp['mb_2008_y']),
Utils.p(woreda_hc['mb_2008_y'] + woreda_hp['mb_2008_y'])
]]
def header_footer(canvas, doc):
# Save the state of our canvas so we can draw on it
canvas.saveState()
# header
canvas.drawImage('assets/images/FMOH.png', doc.leftMargin + 10,
doc.height + doc.topMargin, 48, 48)
canvas.drawImage('assets/images/EPHI.png',
doc.leftMargin + doc.width - 53,
doc.height + doc.topMargin, 48, 48)
header = Utils.p('Malaria Elimination Baseline', 18, True, '#0085B7')
w, h = header.wrap(doc.width, doc.topMargin)
header.drawOn(canvas, doc.leftMargin,
doc.height + doc.topMargin + h + 20)
header = Utils.p('Woreda Factsheet', 16, False, '#0085B7')
w, h = header.wrap(doc.width, doc.topMargin)
header.drawOn(canvas, doc.leftMargin, doc.height + doc.topMargin + h)
canvas.setLineWidth(.3)
#canvas.setStrokeColorRGB(0, 1, 0.3)
canvas.line(doc.leftMargin + 5.5, doc.bottomMargin - 15,
doc.width + 14, doc.bottomMargin - 15)
# Footer
canvas.drawImage('assets/images/SMMES.png', doc.leftMargin + 5,
doc.bottomMargin - 45, 158, 25)
canvas.drawImage('assets/images/WHO.png', doc.leftMargin + 188,
doc.bottomMargin - 45, 63, 23) # 188:- 158 + 25 +5
canvas.drawImage('assets/images/CHAI.png', doc.leftMargin + 276,
doc.bottomMargin - 45, 43, 23) # 276:- 188 + 63 + 25
canvas.drawImage('assets/images/TU.png', doc.leftMargin + 344,
doc.bottomMargin - 45, 63, 23) # 344:- 276 + 43 + 25
canvas.drawImage('assets/images/PATH.png', doc.leftMargin + 432,
doc.bottomMargin - 45, 49, 19) # 432:- 344 + 63 + 50
# Page number
canvas.setFont("mb", 7)
canvas.setFillColor('#0085B7')
page_num = canvas.getPageNumber()
page_num_text = 'Page {}'.format(page_num)
# 524:- 158 + 50 + 43 + 50 + 63 + 50 + 110
canvas.drawRightString(doc.leftMargin + 524, doc.bottomMargin - 40,
page_num_text)
# Release the canvas
canvas.restoreState()
class Latest:
def __init__(self):
self.utils = Utils()
self.command = Command()
self.console = Console()
self.location = Location()
def run(self):
home = str(Path.home())
stream = self.utils.load_yaml(home + self.location.GITHUB +
self.location.REPOSITORIES_FILE)
for repo in stream:
print()
code = repo.get(self.command.CODE)
self.console.print("Processing [bold red]" + code + "[/]...")
r = requests.get('https://api.github.com/repos/' + code +
"/releases/latest")
response = r.json()
self.console.print("[green]URL[/]: [bold magenta]" +
response["html_url"] + "[/]")
self.console.print("[green]Tag[/]: [bold magenta]" +
response["tag_name"] + "[/]")
self.console.print("[green]Publish date[/]: [bold magenta]" +
response["published_at"] + "[/]")
self.console.print("[green]Release name[/]: [bold magenta]" +
response["name"] + "[/]")
def __init__(self):
self.config = self.read_config()
self.utils = Utils(self.config)
self.pi_utils = PiUtils(self.config, self.utils)
self.eve = evelink.eve.EVE()
self.db = Connect(self.config['database']['data'])
self.classes = self.db.base.classes
parser = argparse.ArgumentParser()
parser.add_argument('--create-db', help='create the databases', action='store_true')
parser.add_argument('--pi', help='update the Planetary Interaction cache', action='store_true')
parser.add_argument('--losses', help='update the items and ships that have been destroyed',
action='store', type=int)
parser.add_argument('--alliances', help='update the alliances', action='store_true')
self.args = parser.parse_args()
if self.args.create_db:
self.create_databases()
if self.args.pi:
self.update_pi()
if self.args.losses:
self.update_losses()
if self.args.alliances:
self.update_alliances()
def kline(codes, scale=240, ma=5, length=1023):
args = (scale, ma, length)
url = 'http://money.finance.sina.com.cn/quotes_service/api/json_v2.php/' \
'CN_MarketData.getKLineData?symbol={}&scale={}&ma={}&datalen={}'
urls = list(map(
lambda x: url.format('sh'+x[-6:], *args) if x[0]=='6' else \
url.format('sz'+x[-6:], *args),
codes
))
# loop = asyncio.get_event_loop()
# results = loop.run_until_complete(Utils.fetch_all(urls))
results = Utils.fetch_all(urls)
klines = list(
map(
lambda x: json.loads(
re.sub('(\w+)\s?:\s?("?[^",]+"?,?)', "\"\g<1>\":\g<2>", x)
), results))
stocks = {}
for idx, code in enumerate(codes):
stocks[code] = klines[idx]
return stocks
def login_user():
if request.method == 'POST':
username = request.form['username']
password = request.form['password']
user = UserModel.find_by_username(username)
try:
#if user and safe_str_cmp(password, user.password):
if user and Utils.check_hashed_password(password, user.password):
session['username'] = user.username
if user.username in current_app.config.get('ADMIN',
'').split(','):
session['admin'] = True
else:
session['admin'] = None
return redirect(url_for("webmodels.index"))
else:
session['username'] = None
session['admin'] = None
return "Invalid credentials! please try again."
except:
session['username'] = None
session['admin'] = None
return "Invalid credentials! please try again."
return render_template("users/login.html")
def post(self):
json_data = request.get_json()
if not json_data:
return {'message': 'No input data provided'}
user_data = user_schema.load(json_data)
if not UserModel.find_by_username(username=user_data['username']):
return {'message': 'Username is not exists'}
user_ = user_schema.dump(UserModel.find_by_username(username=user_data['username']))
if Utils.decrypt_pass(user_data['password'], user_['password']):
access_token = create_access_token(user_['username'])
refresh_token = create_refresh_token(user_['username'])
return {
'message': "Login successful!",
'access_token': access_token,
'refresh_token': refresh_token
}
else:
return {
'message': 'Username or password wrong!'
}
def real(self, codes, group_count=None, array=None):
length = len(codes)
codes_with_prefix = list(
map(lambda x: 'sh' + x[-6:]
if x[0] == '6' else 'sz' + x[-6:], codes))
if group_count is None:
group_size = self.max_group_size
group_count = math.ceil(length / group_size)
else:
min_group_count = math.ceil(length / self.max_group_size)
if group_count < min_group_count:
group_count = min_group_count
group_size = math.ceil(length / group_count)
code_lists = list([
','.join(codes_with_prefix[idx *
group_size:min((idx + 1) *
group_size, length)])
for idx in range(self.group_count)
])
urls = [self.stock_api + stock_list for stock_list in code_lists]
# loop = asyncio.get_event_loop()
# result = loop.run_until_complete(Utils.fetch_all(urls))
result = Utils.fetch_all(urls)
return self.parse_real_data(''.join(result),
len(self.codes),
array=array)
def set_basics(self):
# set zt_price
returns = self.send_processes_message(self.hq_pipes, {
'type': 'function',
'function': 'check',
'args': [],
'kwargs': {}
})
idx = returns[0]['msl']
self.basics.data[:, 0] = np.around(self.msl.data[idx, :, 1] * 1.1,
decimals=2)
self.basics.data[:, 1] = np.around(self.msl.data[idx, :, 1] * 0.9,
decimals=2)
# set ma5vpm
self.calculate_ma5vpm()
# set ltgb
dt = Utils.previous_trading_day(datetime.now())
df = TradingDay.ts.daily_basic(trade_date=dt.strftime('%Y%m%d'))
for index, code in enumerate(self.codes):
scode = code + '.SH' if code[0] == '6' else code + '.SZ'
result = df.loc[df['ts_code'] == scode, 'float_share']
if len(result) == 0:
continue
self.basics.data[index, 3] = result.iloc[0]
def get_lastest_stock_codes(self):
old_codes = Utils.get_stock_codes()
code_filter = Utils.code_filter
step_size = 1000
codes = []
for market in Utils.markets:
count = self.tdx.get_security_count(market)
print(market, count, end=' : ')
steps = math.ceil(count / step_size)
total = 0
for step in range(steps):
result = self.tdx.get_security_list(market, step_size * step)
print(str(step) + '/' + str(steps), end=", ")
for item in result:
code = item['code'].strip()
if code[:3] in code_filter[market]:
codes.append(code)
total += 1
market_name = '深市' if market == 0 else '沪市'
print(market_name + ' A股 总数: ' + str(total))
print('沪深 A股 总数:' + str(len(codes)))
return codes
def get(self, id):
if not id:
return False
sql = "SELECT %s FROM %s WHERE id = %d"
result = self._db.select(
sql, [','.join(self._selectItens), self._table, id])
if result['err'] is True:
self.__setError(result['ret'])
return Utils.structReturn(True, 0)
else:
if result['ret']['total'] > 0:
return Utils.structReturn(False, result['ret']['itens'])
else:
self.__setError("Nenhum dado encontrado!")
return Utils.structReturn(True, 0)
def execute(self, cmd, params = ()):
result = None
try:
newParams = ()
for item in params:
newParams = newParams + (self.__conn.escape_string(item) if isinstance(item, str) else item,)
cur = self.__conn.cursor()
cur.execute(cmd % newParams)
self.__conn.commit()
result = Utils.structReturn(False, {'total': cur.rowcount if hasattr(cur, 'rowcount') else 0, 'id': self.__conn.lastrowid if hasattr(self.__conn,'lastrowid') else 0})
except MySQLdb.Error, e:
result = Utils.structReturn(True, "Error %d: %s" % (e.args[0],e.args[1]))
def parseById(self, tipoVeiculo, id):
for sel in self._DOMtree.xpath('//select[@id="%s"]/option' % (id)):
text = sel.xpath('text()').extract()
value = sel.xpath('@value').extract()
if type(text).__name__ == 'list':
text = text[0]
if type(value).__name__ == 'list':
value = value[0]
value = int(Utils.toUTF8(value))
text = str(Utils.toUTF8(text))
print self._model.add(value, tipoVeiculo, text)
# endfor
# enddef
def reset_suspended_stock_codes():
now = datetime.now()
time1 = now.replace(hour=8, minute=45, second=0, microsecond=0)
time2 = now.replace(hour=15, minute=0, second=0, microsecond=0)
if time1 < now < time2:
print('请在8:30之前 或 15:05之后, 初始化 停牌股票')
return
suspended_codes = []
market_snapshot = Sina(Utils.get_stock_codes()).market_snapshot()
for code in market_snapshot:
stock = market_snapshot[code]
if stock['turnover'] == 0 and stock['close'] != 0:
suspended_codes.append(code)
Utils.save_suspended_stock_codes(suspended_codes)
return suspended_codes
def market_snapshot(self, array=None):
urls = [self.stock_api + stock_list for stock_list in self.code_lists]
# loop = asyncio.get_event_loop()
# result = loop.run_until_complete(Utils.fetch_all(urls))
result = Utils.fetch_all(urls)
return self.parse_real_data(''.join(result),
len(self.codes),
array=array)
def parseById(self, tipoVeiculo, id):
for sel in self._DOMtree.xpath('//select[@id="%s"]/option' % (id)):
text = sel.xpath('text()').extract()
value = sel.xpath('@value').extract()
if type(text).__name__ == 'list':
text = text[0]
if type(value).__name__ == 'list':
value = value[0]
value = int(Utils.toUTF8(value))
text = str(Utils.toUTF8(text))
print self._model.add(value, tipoVeiculo, text)
# endfor
# enddef
def execute(self, response):
metas = response.meta
try:
jsonValue = json.loads(response.body)
print response.url
print response.status
print len(jsonValue)
print "---"
if 'erro' in jsonValue:
print jsonValue
print "Deu erro :("
else:
countSaved = 0
CodigoFipe = Utils.toUTF8(jsonValue['CodigoFipe'])
Modelo = Utils.toUTF8(jsonValue['Modelo'])
Valor = Utils.toUTF8(jsonValue['Valor'])
try:
Valor = float((re.sub("[^0-9,]+", "", Valor)).replace(",","."))
except e:
Valor = 0
if Valor>0:
print self._model.add(CodigoFipe, metas['codigoAnoModelo'], metas['codigoModelo'], metas['codigoMarca'], metas['codigoTipoVeiculo'], metas['codigoTabelaReferencia'], Modelo, Valor)
countSaved = countSaved + 1
if countSaved > 0:
anoModelo = AnoModeloModel(self._db)
anoModelo.setCrawled(metas['codigoTipoVeiculo'], metas['codigoTabelaReferencia'], metas['codigoMarca'], metas['codigoModelo'], metas['codigoAnoModelo'])
except e:
print 'Except =>'
print e
# enddef
class Command():
def __init__(self):
self.config = self.read_config()
self.utils = Utils(self.config)
self.pi_utils = PiUtils(self.config, self.utils)
self.eve = evelink.eve.EVE()
self.db = Connect(self.config['database']['data'])
self.classes = self.db.base.classes
parser = argparse.ArgumentParser()
parser.add_argument('--create-db', help='create the databases', action='store_true')
parser.add_argument('--pi', help='update the Planetary Interaction cache', action='store_true')
parser.add_argument('--losses', help='update the items and ships that have been destroyed',
action='store', type=int)
parser.add_argument('--alliances', help='update the alliances', action='store_true')
self.args = parser.parse_args()
if self.args.create_db:
self.create_databases()
if self.args.pi:
self.update_pi()
if self.args.losses:
self.update_losses()
if self.args.alliances:
self.update_alliances()
def read_config(self, path='config.json'):
with open(path) as cfg:
config = json.load(cfg)
assert(config)
return config
def create_databases(self):
from sql import initialize_sql
from sql.pi import Pi
from sql.losses import ItemsLost, Kills
from sql.alliances import Corporation, Alliance
initialize_sql(self.db.engine)
def update_pi(self):
print('updating PI stats...')
for system_name in self.config['statistics']['pi_systems']:
system = self.utils.lookup_system(system_name).__dict__
for tier in self.config['statistics']['pi_tiers']:
self.pi_utils.store_prices(tier, system['solarSystemID'])
def update_losses(self):
'''Pull the kills from zKillboard and store them in the database'''
print('updating losses...')
losses = self.db
items_lost = {}
alliance_ids = []
alliances_requested = []
time_format = '%Y-%m-%d %H:%M:%S'
start_time = (datetime.datetime.now() + relativedelta(months=-1)).strftime('%Y%m%d%H%I')
# add alliance ids from coalitions to alliance_ids
for coalition in self.config['coalitions']:
alliance_ids.extend(self.config['coalitions'][coalition])
for alliance_id in alliance_ids:
# what?
if alliance_id in alliances_requested:
continue
alliances_requested.append(alliance_id)
kb_url = 'https://zkillboard.com/api/kills/allianceID/%s/startTime/%s' % (alliance_id, start_time)
# USE REQUESTS JSON
page = requests.get(kb_url)
data = json.loads(page.text)
print('Working on: %s' % (alliance_id))
# Example
#{"characterID":926924867,"characterName":"clavo oxidado",
#"corporationID":98390683,"corporationName":"Hogyoku","allianceID":1354830081,
#"allianceName":"Goonswarm Federation","factionID":0,"factionName":"","securityStatus":-0.9,"damageDone":0,"finalBlow":0,"weaponTypeID":5439,"shipTypeID":0}
# https://zkillboard.com/kill/54204593/
for row in data:
kill_time = datetime.datetime.strptime(row['killTime'], time_format)
kill_id = row['killID']
# See if the killID already exists in the database
query = losses.session.query(losses.base.classes.kills).filter_by(killID=kill_id).first()
if query:
print('killID already exists, skipping...')
continue
# The kill itself
kill = losses.base.classes.kills(killID=kill_id,
shipTypeID=row['victim']['shipTypeID'],
killTime=kill_time,
characterID=row['victim']['characterID'],
corporationID=row['victim']['corporationID'],
corporationName=row['victim']['corporationName'],
allianceID=row['victim']['allianceID'])
# Attach the items lost to the kill
for line in row['items']:
item = losses.base.classes.items_lost(typeID=line['typeID'])
kill.items_lost_collection.append(item)
for line in row['attackers']:
attacker = losses.base.classes.attacker(weaponTypeID=line['weaponTypeID'],
allianceID=line['allianceID'],
corporationName=line['corporationName'],
shipTypeID=line['shipTypeID'],
characterName=line['characterName'],
characterID=line['characterID'],
allianceName=line['allianceName'])
kill.attacker_collection.append(attacker)
losses.session.add(kill)
losses.session.commit()
def update_alliances(self):
#{'id': 99000304, 'member_count': 8, 'ticker': 'PIGS', 'name': 'THE SPACE P0LICE', 'executor_id': 98038175, 'member_corps': {699225816: {'id': 699225816, 'timestamp': 1423864860}, 875152542
print('Deleting alliances and corporations...')
self.db.session.query(self.db.base.classes.alliances).delete()
self.db.session.query(self.db.base.classes.corporations).delete()
self.db.session.commit()
print('Adding current alliances...')
alliances = self.eve.alliances().result
for id_ in alliances.keys():
member_count = alliances[id_]['member_count']
ticker = alliances[id_]['ticker']
name = alliances[id_]['name']
executor_id = alliances[id_]['executor_id']
member_corps = alliances[id_]['member_corps']
alliance = self.db.base.classes.alliances(id=id_,
member_count=member_count,
ticker=ticker,
name=name,
executor_id=executor_id)
for c in member_corps.keys():
print(member_corps[c])
corp = self.db.base.classes.corporations(alliance_id=member_corps[c]['id'])
alliance.corporations_collection.append(corp)
self.db.session.add(alliance)
self.db.session.commit()