def main():
#fname = 'lessions_names.txt'
fname = 'lessions_html_code.txt'
sets = iow.get_utf8_template()
sets['name'] = fname
i = 0
result_list = []
readed_list = iow.file2list(sets)
for at in readed_list:
# Предварительная фильтарция
at = at.replace('</a>', '')
# Ссылки с содержанием
if 'pdf' in at or '&format=srt' in at:
at_copy = at.replace(' <a target="_new" href=', '')
#result_list.append('link_to_get '+_split_url(at_copy))
result_list.append(_split_url(at_copy))
# Темы
if 'min)' in at and 'div' not in at:
result_list.append('name_content '+at)
# Части-недели
if '(Week' in at:
at_copy_list = at.split(' ')[1].split('</h3>')[0]
result_list.append('folder '+at_copy_list)
i += 1
# теперь нужно запаковать в словарь
levels = ['folder', 'name_content', 'link_to_get']
result = {}
for at in result_list:
print at
_fill_tree_dict(result_list, levels, 0, result, 'root')
#_tree_walker(result)
# сохраняем результаты в файл
to_file = [json.dumps(result, sort_keys=True, indent=2)]
settings = {
'name': 'extracted_from_html.json',
'howOpen': 'w',
'coding': 'cp866' }
iow.list2file(settings, to_file)
settings = {
'name': 'result.list',
'howOpen': 'w',
'coding': 'cp866' }
iow.list2file(settings, result_list)
def test_shuffle(self):
sets = iow.get_utf8_template()
sets['name'] = 'result_test.list'
readed_list = iow.file2list(sets)
# теперь нужно запаковать в словарь
levels = ['LEVEL0', 'LEVEL1', 'nothing']
result = {}
_fill_tree_dict(readed_list, levels, 0, result, 'root')
json_result = json.dumps(result['root'], sort_keys=True, indent=2)
print
print json_result
def _read_cp1251_file(fname):
"""
Returns:
Возвращает список и ответ на вопрос - угадали ли мы кодировку?
"""
this_is_it = True
sets = iow.get_utf8_template()
sets['coding'] = 'cp1251'
sets['name'] = fname
# Если не та кодировка - возвращает пустой список
readed_list = iow.file2list(sets)
#if not readed_list:
# this_is_it = False
# Не все находит! Нужно еще построчно
if _list_is_utf8(readed_list):
this_is_it = False
return readed_list, this_is_it
def start_process(self,fname):
# Читаем результат
print 'Reading file..'
sets = iow.get_utf8_template()
sets['name'] = fname
readed_list = iow.file2list(sets)
# Обрабатываем результат
print 'Start processing..'
separ_sign = '@@@'
tmp_string = separ_sign.join(readed_list)
tmp_list = tmp_string.split('top - ')
result_list = []
for it in tmp_list:
result_list.append(it.replace(separ_sign, ' '))
result_list = result_list[1:]
#self._print_list(result_list)
#return result_list
# Формирование словаря
print 'Forming dictionary..'
num_list = len(result_list)
result_dict = {'time': range(num_list),
'cpu_us': range(num_list),
'cpu_sy': range(num_list),
'cpu_ni': range(num_list),
'cpu_id': range(num_list),
'cpu_wa': range(num_list),
'cpu_hi': range(num_list),
'cpu_si': range(num_list),
'cpu_st': range(num_list),
'mem_total': range(num_list),
'mem_used': range(num_list),
'mem_free': range(num_list),
'mem_buffers': range(num_list)}
# Вытаскивание значений, исходя из особенностей их расположения
def _get_cpu(num,type_string):
position_num = result_list[num].find('%'+type_string)
return result_list[num][position_num-4:position_num].replace(' ','')
def _get_mem(num,type_string):
mem_string = result_list[num][result_list[num].find('Mem'):
result_list[num].find('Swap')]
position_num = mem_string.find(type_string)
founded_string = mem_string[position_num-10:position_num-2]
founded_string = founded_string.replace(' ','')
return founded_string
# Запись в словарь
print 'Writing results in dictionary..'
k = 0
for at in result_list:
result_dict['time'][k] = result_list[k][0:8]
result_dict['cpu_us'][k] = _get_cpu(k,'us')
result_dict['cpu_sy'][k] = _get_cpu(k,'sy')
result_dict['cpu_ni'][k] = _get_cpu(k,'ni')
result_dict['cpu_id'][k] = _get_cpu(k,'id')
result_dict['cpu_wa'][k] = _get_cpu(k,'wa')
result_dict['cpu_hi'][k] = _get_cpu(k,'hi')
result_dict['cpu_si'][k] = _get_cpu(k,'si')
result_dict['cpu_st'][k] = _get_cpu(k,'st')
result_dict['mem_total'][k] = _get_mem(k,'total')
result_dict['mem_used'][k] = _get_mem(k,'used')
result_dict['mem_free'][k] = _get_mem(k,'free')
result_dict['mem_buffers'][k] = _get_mem(k,'buffers')
k+=1
#self._print_dict(result_dict)
# Преобразование в json-строку
print 'To JSON. Result: '
print self.dict_to_json(result_dict)
# Преобразование в csv-строку
#print dict_to_csv(result_dict)
# Вывод в файл .csv - таблицы
print 'Write csv-table in', self.out_file_csv_r
table_csv_string = self.dict_to_csv_r(result_dict)
sets = iow.get_utf8_template()
sets['name'] = self.out_file_csv_r
sets['howOpen'] = 'w'
#file_obj = open('param_table.csv','w')
iow.app_str(sets, table_csv_string)
return []
def _tree_walker(tree, i):
if not tree or 'list' in str(type(tree)):
return
# Выводим заголовки
for key in tree:
step = ''
for at in range(i):
step += '\t'
print step+key
_tree_walker(tree[key], i+1)
if __name__ == '__main__':
sets = iow.get_utf8_template()
sets['name'] = 'D:/home/lugansky-igor/web-pc-mc-chain-framework/mkts/system_settings.mkt'
readed_list = iow.file2list(sets)
folded_list = []
for at in readed_list:
if at and not _all_symbols_is_spaces(at):
if at[0] != '#':
# Чистые данные
if at[0] != ' ':
at = 'LEVEL0 '+at
else:
at = at.replace(' ', '@')
folded_list.append(at)
def _cp1251_list_to_file(list_lines, fname):
sets = iow.get_utf8_template()
sets['name'] = fname
sets['coding'] = 'cp1251'
sets['howOpen'] = 'w'
iow.list2file(sets, list_lines)
def _utf8_file_to_list(fname):
sets = iow.get_utf8_template()
sets['coding'] = 'utf_8'
sets['name'] = fname
return iow.file2list(sets)
