def get_second_page_data(debug=False): """#get second page data, and save result to file ./api-data/second_page_data.json""" t0 = time.time() helper.remove_file(config.FILENAME_SECOND_PAGE_DATA) helper.remove_file(config.FILENAME_SECOND_PAGE_SECTIONIDS_DATA) helper.create_dir(config.SAVE_FILE_DIR) helper.my_print("total has %d grades...\n" % len(config.GRADES), debug) for grade in config.GRADES: time.sleep(config.SLEEP_TIME) helper.my_print("%s:" % grade, debug) url = helper.get_second_page_url(grade=grade) content = helper.get_url_content(url) # 获取知识章节列表 if content is None: helper.my_print_error("the content is None.") else: helper.save_content_to_file(config.FILENAME_SECOND_PAGE_DATA, content) helper.my_print(content, debug) # 获取知识章节对应的ID列表 json_data = json.loads(content, encoding='utf-8') for course_list in json_data['list']: for course in course_list['list']: section_id = course['courseSectionID'] helper.save_content_to_file( config.FILENAME_SECOND_PAGE_SECTIONIDS_DATA, section_id) helper.my_print(section_id, debug) helper.my_print("", debug) print('Done. %f seconds cost.' % (time.time() - t0))
def main(): args = get_args() os.makedirs(args.csv_converted_folder, exist_ok=True) files = glob(os.path.join(args.csv_folder, '*.csv')) for file in files: name = get_filename(file, with_extension=False) print(name) new_filename = os.path.join(args.csv_converted_folder, name + '.csv') lines = get_csv(file) if not rows_are_valid(lines): print('\t (Skipping)') remove_file(new_filename) continue converted_lines = list(map(convert_line, lines)) write_csv(new_filename, converted_lines) print()
def main(): args = get_args() os.makedirs(args.csv_converted_folder, exist_ok=True) files = glob(os.path.join(args.csv_folder, '*.csv')) for file in files: name = get_filename(file, with_extension=False) print(name) new_filename = os.path.join(args.csv_converted_folder, name + '.csv') lines = get_csv(file) if not rows_are_valid(lines): print('\t (Skipping)') remove_file(new_filename) continue converted_lines = list(map(convert_line, lines)) write_csv(new_filename, converted_lines) print()
def get_main_page_data(debug=False): """#get main page data, and save the result to file ./api-data/main_page_data.json""" helper.remove_file(config.FILENAME_MAIN_PAGE_DATA) # 先删除可能已存在的文件后, 再重新保存 helper.create_dir(config.SAVE_FILE_DIR) t0 = time.time() url = helper.get_main_page_url() content = helper.get_url_content(url) # 获取首页的一级页面列表(包含年级列表,滚动图列表,学习计划列表等) if content is None: helper.my_print_error("main_page_data is None.") else: helper.save_content_to_file(config.FILENAME_MAIN_PAGE_DATA, content) helper.my_print(content, debug) print('Done. %f seconds cost.' % (time.time() - t0))
def get_third_page_data(debug=False): """#get third page data, and save result to file ./api-data/third_page_data.json""" t0 = time.time() helper.remove_file(config.FILENAME_THIRD_PAGE_DATA) helper.remove_file(config.FILENAME_THIRD_SECTIONIDS_DATA) helper.create_dir(config.SAVE_FILE_DIR) with open(config.FILENAME_SECOND_PAGE_SECTIONIDS_DATA) as f: i = 0 lines = f.readlines() helper.my_print("total has %d chapters...\n" % len(lines), debug) for line in lines: i += 1 section_id = str(int(line)) helper.my_print("line:%d sectionID:%s" % (i, section_id), debug) time.sleep(config.SLEEP_TIME) url = helper.get_third_page_url(section_id) content = helper.get_url_content( url) # 根据某个章节的 sectionID, 获取其知识点列表 if content is None: helper.my_print_error("the content is None.") else: helper.save_content_to_file(config.FILENAME_THIRD_PAGE_DATA, content) helper.my_print(content, debug) # 获取知识点对应的课程ID列表(用于根据课程ID, 获取题目列表) json_data = json.loads(content) for course in json_data['list']: course_dic = { 'courseSectionID': course['courseSectionID'], 'sectionName': course['sectionName'], 'parentID': course['parentID'] } data = json.dumps(course_dic, ensure_ascii=False) helper.save_content_to_file( config.FILENAME_THIRD_SECTIONIDS_DATA, data) helper.my_print(data, debug) helper.my_print("", debug) print('Done. %f seconds cost.' % (time.time() - t0))
def main(): # 根据程序名称获取pid pid = helper.get_pid(cfg) # 若pid不存在,则进程已经停止 if not pid: helper.remove_file(cfg.pid_lock()) return "{0} not run".format(cfg.name()) # 关闭进程 helper.stop_proc(pid) # 若pid不存在,则进程停止成功 if not helper.get_pid(cfg): helper.remove_file(cfg.pid_lock()) return "{0}({1}) stop success".format(cfg.name(), pid) else: return "{0}({1}) stop failed".format(cfg.name(), pid)
def main(): if not os.path.exists("restart.py"): return "restart.py not exist" crontab_tmp = "./crontab_tmp" helper.crontab_to_file(crontab_tmp) try: line_numbers = helper.grep_line_numbers(cfg.timer_commamd(), crontab_tmp) if len(line_numbers) == 0: return "timer not install" else: for number in line_numbers: helper.remove_file_line(number, crontab_tmp) helper.crontab_load_file(crontab_tmp) return "timer remove success" finally: helper.remove_file(crontab_tmp)
def main(): if not os.path.exists("restart.py"): return "restart.py not exist" crontab_tmp = "./crontab_tmp" helper.crontab_to_file(crontab_tmp) try: line_numbers = helper.grep_line_numbers(cfg.timer_commamd(), crontab_tmp) if len(line_numbers) != 0: return "timer already installed" # 添加定时重启命令 helper.append_file(cfg.timer_commamd(), crontab_tmp) # 将定时脚本加载到crontab if helper.crontab_load_file(crontab_tmp): return "timer install success" else: return "timer install failed" finally: helper.remove_file(crontab_tmp)
print helper.coloured_output("Uploaded new file done" , 'green') #print file_result print helper.coloured_output("Getting list of children files" , 'yellow') children_files = helper.retrieve_all_files( drive_service, config['backup_folder_id'] ) # print "children_files", children_files #print children_files print "This folder have {0} files".format( str(len(children_files)) ) if len( children_files ) > config['max_file_in_folder']: #Remove old backup file number_delete_file = len(children_files) - config['max_file_in_folder'] count = 0 index_delete_file = len(children_files) -1 while count < number_delete_file: children_id = children_files[index_delete_file]['id'] print helper.coloured_output( "Removing old file with id " + children_id , 'yellow') helper.remove_file( drive_service, children_id ) count +=1 index_delete_file -=1 print helper.coloured_output("Done backup file to google drive !" , 'green') except Exception, e: print "error" print e finally: pass
print(helper.coloured_output("Getting list of children files", 'yellow')) children_files = helper.retrieve_all_files(drive_service, config['backup_folder_id']) # print("children_files", children_files) #print(children_files) print("This folder have {0} files".format(str(len(children_files)))) if len(children_files) > config['max_file_in_folder']: #Remove old backup file number_delete_file = len(children_files) - config['max_file_in_folder'] count = 0 index_delete_file = len(children_files) - 1 while count < number_delete_file: children_id = children_files[index_delete_file]['id'] print( helper.coloured_output( "Removing old file with id " + children_id, 'yellow')) helper.remove_file(drive_service, children_id) count += 1 index_delete_file -= 1 print(helper.coloured_output("Done backup file to google drive !", 'green')) except Exception as e: print("error") print(e) finally: pass
def get_question_data(debug=False, start=1, count=0): """#get question data, and save result to file ./api-data/question_data.json""" t0 = time.time() with open(config.FILENAME_THIRD_SECTIONIDS_DATA) as f: i = 0 lines = f.readlines() line_count = len(lines) # 检查开始索引参数 start try: start_index = int(start) except ValueError as e: helper.my_print_error( 'Error: %s\n the "start" param must be a Integer number!' % e) return else: if start_index < 1 or start_index > line_count: helper.my_print_error( 'Error: the "start" param must in range(1, len(lines)+1)') return # 检查获取条目参数 count try: limit_count = int(count) except ValueError as e: helper.my_print_error( 'Error: %s\n the "count" param must be a Integer number!' % e) return else: if limit_count <= 0 or (start_index - 1 + limit_count) > line_count: limit_count = line_count - start_index + 1 if start_index == 1: # 当 start_index 值为1时, 表示重新开始获取, 此时应先清除之间存在的数据文件 helper.remove_file(config.FILENAME_QUESTION_DATA) helper.create_dir(config.SAVE_FILE_DIR) helper.my_print("total has %d sections...\n" % line_count, debug) for line in lines: i += 1 if i < start_index: continue elif i >= (start_index + limit_count): break json_data = json.loads(line) section_id = json_data['courseSectionID'] helper.my_print("line:%d sectionID:%s" % (i, section_id), debug) time.sleep(config.SLEEP_TIME) url = helper.get_question_url(section_id) content = helper.get_url_content(url) # 根据知识点的 sectionID, 获取题目对表 if content is None: # 若发生了异常时, 直接退出 helper.my_print_error("the content is None.") helper.my_print_error( "An Exception has happen. get_question_data will be exit. " "the next start_index should be %d" % i) break else: result_data = json.loads(content) if result_data['code'] != '99999': # 若返回码不是表示"成功"时, 直接退出 helper.my_print_error( "result code is not 99999. get_question_data will be exit. " "the next start_index should be %d" % i) break helper.save_content_to_file(config.FILENAME_QUESTION_DATA, content) helper.my_print(content, debug) helper.my_print("", debug) print('Done. %f seconds cost.' % (time.time() - t0))