def main(): #fname = 'lessions_names.txt' fname = 'lessions_html_code.txt' sets = iow.get_utf8_template() sets['name'] = fname i = 0 result_list = [] readed_list = iow.file2list(sets) for at in readed_list: # Предварительная фильтарция at = at.replace('</a>', '') # Ссылки с содержанием if 'pdf' in at or '&format=srt' in at: at_copy = at.replace(' <a target="_new" href=', '') #result_list.append('link_to_get '+_split_url(at_copy)) result_list.append(_split_url(at_copy)) # Темы if 'min)' in at and 'div' not in at: result_list.append('name_content '+at) # Части-недели if '(Week' in at: at_copy_list = at.split(' ')[1].split('</h3>')[0] result_list.append('folder '+at_copy_list) i += 1 # теперь нужно запаковать в словарь levels = ['folder', 'name_content', 'link_to_get'] result = {} for at in result_list: print at _fill_tree_dict(result_list, levels, 0, result, 'root') #_tree_walker(result) # сохраняем результаты в файл to_file = [json.dumps(result, sort_keys=True, indent=2)] settings = { 'name': 'extracted_from_html.json', 'howOpen': 'w', 'coding': 'cp866' } iow.list2file(settings, to_file) settings = { 'name': 'result.list', 'howOpen': 'w', 'coding': 'cp866' } iow.list2file(settings, result_list)
def test_shuffle(self): sets = iow.get_utf8_template() sets['name'] = 'result_test.list' readed_list = iow.file2list(sets) # теперь нужно запаковать в словарь levels = ['LEVEL0', 'LEVEL1', 'nothing'] result = {} _fill_tree_dict(readed_list, levels, 0, result, 'root') json_result = json.dumps(result['root'], sort_keys=True, indent=2) print print json_result
def _read_cp1251_file(fname): """ Returns: Возвращает список и ответ на вопрос - угадали ли мы кодировку? """ this_is_it = True sets = iow.get_utf8_template() sets['coding'] = 'cp1251' sets['name'] = fname # Если не та кодировка - возвращает пустой список readed_list = iow.file2list(sets) #if not readed_list: # this_is_it = False # Не все находит! Нужно еще построчно if _list_is_utf8(readed_list): this_is_it = False return readed_list, this_is_it
def start_process(self,fname): # Читаем результат print 'Reading file..' sets = iow.get_utf8_template() sets['name'] = fname readed_list = iow.file2list(sets) # Обрабатываем результат print 'Start processing..' separ_sign = '@@@' tmp_string = separ_sign.join(readed_list) tmp_list = tmp_string.split('top - ') result_list = [] for it in tmp_list: result_list.append(it.replace(separ_sign, ' ')) result_list = result_list[1:] #self._print_list(result_list) #return result_list # Формирование словаря print 'Forming dictionary..' num_list = len(result_list) result_dict = {'time': range(num_list), 'cpu_us': range(num_list), 'cpu_sy': range(num_list), 'cpu_ni': range(num_list), 'cpu_id': range(num_list), 'cpu_wa': range(num_list), 'cpu_hi': range(num_list), 'cpu_si': range(num_list), 'cpu_st': range(num_list), 'mem_total': range(num_list), 'mem_used': range(num_list), 'mem_free': range(num_list), 'mem_buffers': range(num_list)} # Вытаскивание значений, исходя из особенностей их расположения def _get_cpu(num,type_string): position_num = result_list[num].find('%'+type_string) return result_list[num][position_num-4:position_num].replace(' ','') def _get_mem(num,type_string): mem_string = result_list[num][result_list[num].find('Mem'): result_list[num].find('Swap')] position_num = mem_string.find(type_string) founded_string = mem_string[position_num-10:position_num-2] founded_string = founded_string.replace(' ','') return founded_string # Запись в словарь print 'Writing results in dictionary..' k = 0 for at in result_list: result_dict['time'][k] = result_list[k][0:8] result_dict['cpu_us'][k] = _get_cpu(k,'us') result_dict['cpu_sy'][k] = _get_cpu(k,'sy') result_dict['cpu_ni'][k] = _get_cpu(k,'ni') result_dict['cpu_id'][k] = _get_cpu(k,'id') result_dict['cpu_wa'][k] = _get_cpu(k,'wa') result_dict['cpu_hi'][k] = _get_cpu(k,'hi') result_dict['cpu_si'][k] = _get_cpu(k,'si') result_dict['cpu_st'][k] = _get_cpu(k,'st') result_dict['mem_total'][k] = _get_mem(k,'total') result_dict['mem_used'][k] = _get_mem(k,'used') result_dict['mem_free'][k] = _get_mem(k,'free') result_dict['mem_buffers'][k] = _get_mem(k,'buffers') k+=1 #self._print_dict(result_dict) # Преобразование в json-строку print 'To JSON. Result: ' print self.dict_to_json(result_dict) # Преобразование в csv-строку #print dict_to_csv(result_dict) # Вывод в файл .csv - таблицы print 'Write csv-table in', self.out_file_csv_r table_csv_string = self.dict_to_csv_r(result_dict) sets = iow.get_utf8_template() sets['name'] = self.out_file_csv_r sets['howOpen'] = 'w' #file_obj = open('param_table.csv','w') iow.app_str(sets, table_csv_string) return []
def _tree_walker(tree, i): if not tree or 'list' in str(type(tree)): return # Выводим заголовки for key in tree: step = '' for at in range(i): step += '\t' print step+key _tree_walker(tree[key], i+1) if __name__ == '__main__': sets = iow.get_utf8_template() sets['name'] = 'D:/home/lugansky-igor/web-pc-mc-chain-framework/mkts/system_settings.mkt' readed_list = iow.file2list(sets) folded_list = [] for at in readed_list: if at and not _all_symbols_is_spaces(at): if at[0] != '#': # Чистые данные if at[0] != ' ': at = 'LEVEL0 '+at else: at = at.replace(' ', '@') folded_list.append(at)
def _cp1251_list_to_file(list_lines, fname): sets = iow.get_utf8_template() sets['name'] = fname sets['coding'] = 'cp1251' sets['howOpen'] = 'w' iow.list2file(sets, list_lines)
def _utf8_file_to_list(fname): sets = iow.get_utf8_template() sets['coding'] = 'utf_8' sets['name'] = fname return iow.file2list(sets)