def __init__(self, master): # Prevent the app window from being resized master.resizable(False, False) self.banner = Banner(master) self.content = Content(master) # KEYBOARD SHORTCUTS SECTION master.bind('<Control-n>', lambda e: self.content.add_task()) # MENU BAR SECTION master.option_add('*tearOff', False) self.menubar = Menu(master) master.config(menu=self.menubar) # FILE MENU file = Menu(self.menubar) file.add_command(label='Test', command=lambda: messagebox.showinfo( title='Test', message='This is a test.')) file.add_command(label='Add Task', command=lambda: self.content.add_task()) file.entryconfig('Add Task', accelerator='CTRL+N') self.menubar.add_cascade(menu=file, label='File') # EDIT MENU edit = Menu(self.menubar) edit.add_command(label='Change Banner Color', command=lambda: self.banner.change_banner_color()) self.menubar.add_cascade(menu=edit, label='Edit')
def setup_content_item_image_cat(self): tags = ["kitty", "kitten", "cat", "cute", "pet", "cutie", "animals"] content = Content("telegram", "@image_cat", tags, application="cats") image = ContentImage("image", "random", self.mockinfo.mockdirwfiles, False, "json", self.mockinfo.historyfile) content.add_content_item(image) return content
def __init__(self,**kwargs): params = { 'serviceID' : 0x02, 'userID' : 0x00, 'encodingType' : 0x00, 'length' : 0x00, "userAccessLevel" : 0x00, "userID" : 0x00, "userNameLength" : 0x00, "userName" : "", "systemStatus" : 0x00, "systemStatusExtended": 0x00, "numberOfCounters" :0x00, "counters" : {}, # {'key' = counterID , counter'} "currentNetMaster" : 0x00, 'passcode' : "000000", }; for key in kwargs: if(key in params) != True: raise ValueError("@LogonContent : invalid parameter {0} ".format(key)); else: params[key] = kwargs[key]; if(isinstance(params['passcode'], basestring)) != True: raise ValueError("@LogonContent: expecting passcode to be a string object."); Content.__init__(self,**params);
def sweep_get_content(): sweep = SweepElement(key="sweep", title="Frequency sweep", default_datatype="src_data") cnt = Content() cnt.add(sweep, position=(0, 0)) return cnt
def __init__(self, **kwargs): attributes = { 'panelNumber' : 0x00, 'channel' : ChannelType.MX_LOOP_TYPE, 'loopNumber' : 0xFF, 'pointCategory' : PointCategoryType.padr_PCAT_REAL, 'pointNumber' : 0xFF, 'logicalPointNumber' : 0xFD, 'logicalPointZone' : 0xFE, 'deviceCategory' : DeviceCategoryType.ALL_DEVICE_TYPE, 'group' : ByteCombinator().combineBytes(0x00, 0x01), #Two Bytes 'outputPointStoreState' : 0x03, 'reserved1' : 0x00, 'reserved2' : 0x00, 'multiAreaType' : MultiAreaType.ALL, 'areas' : Content.createSizeArray(29, 0xFF), 'area' : 0xFF, 'reserved' : 0x01, 'deviceType' : devices.pdev_DEVICE_ID_ALL, 'requestType' : 0x01, 'searchType' : PointSearchType.ACTIVE_INPUTS }; for key in kwargs: if ((key in attributes) != True): raise ValueError(key); attributes[key]= kwargs[key]; Content.__init__(self, **attributes);
def interest(): # get the user's tags userTags = Tag().user_tags(graph_db, request.get_cookie(graphstoryUserAuthKey)) # get the tags of user's friends tagsInNetwork = Tag().tags_in_network( graph_db, request.get_cookie(graphstoryUserAuthKey)) # if the user's content was requested if request.query.get('userscontent') == "true": contents = Content().get_user_content_with_tag( graph_db, request.get_cookie(graphstoryUserAuthKey), request.query.get('tag')) # if the user's friends' content was requested else: contents = Content().get_following_content_with_tag( graph_db, request.get_cookie(graphstoryUserAuthKey), request.query.get('tag')) return template('public/templates/graphs/interest/index.html', layout=applayout, userTags=userTags, tagsInNetwork=tagsInNetwork, contents=contents, title="Interest")
def test_copy_is_not_started_when_folder_is_empty(self, mock_terminal): mock_terminal.provide_initial_cwd.return_value = cwd mock_terminal.get_ls.side_effect = mock_return_values content = Content() content.copy_selected() self.assertEqual('', content.path_to_copy)
def setup_content_item_text(self): content = Content("telegram", "@quotes_literature", "", application="quotes") format_rules = [{"\"%s":0},{"\"\n\n- %s":1}] text = ContentText("text", "random", "database", self.mockinfo.mockdb, format_rules, "") content.add_content_item(text) return content
def test_rename_is_called_with_correct_absolute_paths(self, mock_terminal): mock_terminal.provide_initial_cwd.return_value = cwd mock_terminal.get_ls.side_effect = mock_return_values content = Content() content.rename('aaaa', 'bbbb') mock_terminal.move.aassert_called_with(cwd + 'aaaa', cwd + 'bbbb')
def pub_workflow_transition(self, transition, save_children=True): """ Apply a workflow transition to this Folder, and (unless save_children==False) we also want to update "_view" fields of all children in both mongo and elastic by calling their save() method. """ Content.pub_workflow_transition(self, transition) if save_children: save_recursively(self, include_self=False, set_modified=False)
def __init__(self, ngram, dir): self.docID = DocID() self.tokenizer = Tokenizer("ma") self.content = Content() self.ngram = ngram self.docID.load(dir + "docid.pickle") self.content.load(dir + "content.pickle") self.stopwords = self._load_stopwords(STOPWORDS_FILE)
def test_rename_is_not_called_when_folder_is_empty(self, mock_terminal): mock_terminal.provide_initial_cwd.return_value = cwd mock_terminal.get_ls.side_effect = mock_return_values content = Content() content.rename('a', 'b') mock_terminal.move.assert_not_called()
def test_delete_is_not_called_when_folder_is_empty(self, mock_terminal): mock_terminal.provide_initial_cwd.return_value = cwd mock_terminal.get_ls.side_effect = mock_return_values content = Content() content.delete_selected() mock_terminal.delete.assert_not_called()
def __init__(self, standard_screen): self.is_working = True self.standard_screen = standard_screen curses.curs_set(False) curses.init_pair(2, curses.COLOR_CYAN, curses.COLOR_BLACK) self.content = Content() self.input_keys = InputKeys() self.pane_manager = PaneManager(standard_screen)
def __init__(self): self.content = Content() self.db = MysqlControl() self._RST_RE = re.compile(r'\d+') self._ID_RE0 = re.compile(r'shopId:\s*(\d+)') self._ID_RE1 = re.compile(r'shopID:\s*(\d+)') self._NAME_RE = re.compile(r'cityCnName:\s*"(\S+)"') self._PHONE_RE = re.compile(r'电话:</span>\s+.+tel">(\d{3,4}.\d+)') self._REPLY_RE = re.compile(r'J-busi-reply')
def test_delete_calls_delete_with_child_path(self, mock_terminal): mock_terminal.provide_initial_cwd.return_value = cwd mock_terminal.get_ls.side_effect = mock_return_values content = Content() content.main_pane_selected_line_i = 0 content.delete_selected() mock_terminal.delete.assert_called_with(cwd + 'aaaa')
def __init__(self, url, casperJSScript='pastebintext.js'): threading.Thread.__init__(self) self.url = url self.casperJSScript = casperJSScript self.content = Content() self.data = [] rua = Random_user_agent() self.ua = rua.rand() self.result = None
def test_open_terminal_is_called_with_cwd(self, mock_terminal): mock_terminal.provide_initial_cwd.return_value = cwd mock_terminal.get_ls.side_effect = mock_return_values content = Content() content.main_pane_selected_line_i = 0 content.open_new_terminal() mock_terminal.open_new_terminal.assert_called_with(content.cwd)
def setParameter(self, stype, values): if(stype =='counters'): if(self.validate(stype, values)): if(type(values) == dict): counter = self.getParameter(stype); for key in values: counter[key] = values[key]; self._params[stype] = counter; else: Content.setParameter(self, stype, values)
def __init__(self, parent, *args, **kwargs): tk.Frame.__init__(self, parent, *args, **kwargs) self.parent = parent self.parent.title('todo.txt') self.content = Content(self) self.content.grid(row=0, column=0, sticky='nsew') self.settings = Settings(self, self.content) self.parent.configure(menu=self.settings)
def test_new_folder_calls_make_new_folder_with_name(self, mock_terminal): mock_terminal.provide_initial_cwd.return_value = cwd mock_terminal.get_ls.side_effect = mock_return_values content = Content() content.main_pane_selected_line_i = 0 folder_name = 'new_folder_name' content.make_new_folder(folder_name) mock_terminal.make_new_folder.assert_called_with(cwd + folder_name)
def test_copy_changes_path_to_copy_and_copy_removes_source(self, mock_terminal): mock_terminal.provide_initial_cwd.return_value = cwd mock_terminal.get_ls.side_effect = mock_return_values content = Content() content.main_pane_selected_line_i = 0 content.copy_selected() self.assertEqual(cwd + 'aaaa', content.path_to_copy) self.assertFalse(content.copy_removes_source)
def _list(content, args): """ List the available plugins. """ list_content = Content() utils._populate_plugin_list(list_content, pfilter=args.string) if not len(list_content.plugin_list.plugin_list): print("No result found.") return content formatter = ListDisplay(list_content.plugin_list) verbosity = parsers._get_verbosity(args) list_content.display(formatter, verbose=verbosity) return content
def __init__(self, **kwargs): attributes = { 'clientID': 0x0000, #Double bytes } for key in kwargs: if ((key in attributes) != True): raise ValueError(key) attributes[key] = kwargs[key] Content.__init__(self, **attributes)
def parse(self, site, url): """ Extract content from a given page URL """ bs = self.get_page(url) if bs is not None: title = self.safe_get(bs, site.titleTag) body = self.safe_get(bs, site.bodyTag) if title != '' and body != '': content = Content(url, title, body) content.print()
def handle_select_item(self, item, args): content_data = self.game.datastore.contents[item.content] content = Content(self, content_data.title, content_data.text) self.game.datastore.datamodel.backpack[ self.game.datastore.datamodel.level - 1].add(item.content) item.icon_item.set_visible(True) content.start() self.render() self.item_found_sound.play() # Save data self.game.datastore.changed_data = True
def __init__(self, **kwargs): params = { "resynchronisationData": 0x00, "avaliable": 0x00, "resynchronisationSent": 0x00 } for key in kwargs: if key in params: params[key] = kwargs[key] Content.__init__(self, **params)
def main(): print("Running the configurator") # required for travis tests if len(sys.argv) > 2 and sys.argv[-2] == 'scripts/configurator_tests/': sys.argv = [sys.argv[:-2]] args = parsers._config_arg_parser() if args.error: utils.error_level = 1 print("Starting Savu Config tool (please wait for prompt)") _reduce_logging_level() content = Content(level="all" if args.disp_all else 'user') with warnings.catch_warnings(): warnings.simplefilter("ignore") content.failed = utils.populate_plugins(error_mode=args.error, examples=args.examples) comp = Completer(commands=commands, plugin_list=pu.plugins) utils._set_readline(comp.complete) # if file flag is passed then open it here if args.file: commands['open'](content, args.file) print("\n*** Press Enter for a list of available commands. ***\n") utils.load_history_file(utils.histfile) while True: try: in_list = raw_input(">>> ").strip().split(' ', 1) except KeyboardInterrupt: print() continue command, arg = in_list if len(in_list) == 2 else in_list + [''] command = command if command else 'help' if command not in commands: print("I'm sorry, that's not a command I recognise. Press Enter " "for a list of available commands.") else: content = commands[command](content, arg) if content.is_finished(): break print("Thanks for using the application")
def prepare_email_content(email_address, subject): #construct Receiver, Content and Attachment Class r = Receiver() c = Content() a = Attachment() msg = EmailMessage() msg['Subject'] = subject msg['To'] = ', '.join(r.get_receiver()) msg['From'] = email_address msg.set_content(c.get_content()) set_attachment(msg, a.get_attachments()) return msg
def __init__(self, file_path): self.__init_win() content = Content(file_path) self.image_path = os.path.dirname(file_path) self.presentation_content = content.get_content_list() self.currentpath = os.getcwd() try: self.show_str("tP by jinsheng, modified by imcmy. press 'j' to continue", 0, False) self.start() except KeyboardInterrupt: pass self.__reset_background() curses.endwin()
def __init__(self, file_path): self.__init_win() content = Content(file_path) self.image_path = os.path.dirname(file_path) self.presentation_content = content.get_content_list() self.currentpath = os.getcwd() try: self.show_str("tP by jinsheng, press 'j' to continue", 0, False) self.start() except KeyboardInterrupt: pass self.__reset_background() curses.endwin()
def handle_select_item(self, item, args): content_data = self.game.datastore.contents[item.content] item.set_rollover_image(None) item.remove_event_handler(ItemEvent.CLICK, self.handle_select_item) content = Content(self, content_data.title, content_data.text) self.game.datastore.datamodel.backpack[self.game.datastore.datamodel.level - 1].add(item.content) item.icon_item.set_visible(True) content.start() self.render() self.item_found_sound.play() # Save data self.game.datastore.changed_data = True
def __init__(self, file_path): content = Content(file_path) self.image_path = os.path.dirname(file_path) self.presentation_content = content.get_content_list() self.currentpath = os.getcwd() self.screen = curses.initscr() self.screen.keypad(True) curses.noecho() curses.cbreak() self.height, self.width = self.screen.getmaxyx() self.show_str("tP by jinsheng, press 'j' to continue", 0, False) self.start() self.__reset_background() curses.endwin()
def main(): # required for travis tests if len(sys.argv) > 2 and sys.argv[-2] == 'scripts/configurator_tests/': sys.argv = [sys.argv[:-2]] args = parsers._config_arg_parser() if args.error: utils.error_level = 1 print("Starting Savu Config tool (please wait for prompt)") utils.populate_plugins() comp = Completer(commands=commands, plugin_list=pu.plugins) utils._set_readline(comp.complete) content = Content(level="all" if args.disp_all else 'user') # if file flag is passed then open it here if args.file: commands['open'](content, args.file) print("\n*** Press Enter for a list of available commands. ***\n") while True: try: in_list = raw_input(">>> ").strip().split(' ', 1) except KeyboardInterrupt: print() continue command, arg = in_list if len(in_list) is 2 else in_list+[''] command = command if command else 'help' if command not in commands: print("I'm sorry, that's not a command I recognise. Press Enter " "for a list of available commands.") else: content = commands[command](content, arg) if content.is_finished(): break # write the history to the history file utils.readline.write_history_file(utils.histfile) print("Thanks for using the application")
def __init__(self,url,casperJSScript='pastebintext.js'): threading.Thread.__init__(self) self.url=url self.casperJSScript=casperJSScript self.content=Content() self.data=[] rua=Random_user_agent() self.ua=rua.rand() self.result=None
def task_extract_content(extractor, page_key): page = Page.get(page_key) if not page.pending or Content.already_fetched(page.content_url, extractor): info("skipping content extraction [%s] for page %s, as it is not pending or already fetched" % (extractor, page_key)) return try: get_content_extractor(extractor).extract(page) except Exception, e: error("content extraction %s for page:%s failed with error <%s: %s>" % (extractor, page_key, type(e), e)) raise
def search(self, topic, site): """ Searches a given website for a given topic and records all pages found """ bsObj = self.getPage(site.searchUrl+topic) searchResults = bsObj.select(site.resultListing) for result in searchResults: url = result.select(site.resultUrl)[0].attrs["href"] #Check to see whether it's a relative or an absolute URL if(site.absoluteUrl): pageObj = self.getPage(url) else: pageObj = self.getPage(site.url+url) if pageObj is None: print("Something was wrong with that page or URL. Skipping!") return title = self.safeGet(pageObj, site.pageTitle) body = self.safeGet(pageObj, site.pageBody) if title != "" and body != "": content = Content(topic, title, body, url) content.print()
def get_class_schema(cls, request=None): schema = Content.get_class_schema(request) #schema.add(colander.SchemaNode(colander_types.DateUS(), name='dateline', default=today_for_request_tz(request))) schema.add(colander.SchemaNode(colander_types.DateTimeUS(get_timezone_for_request(request)), name='dateline', default=utcnow())) schema.add(colander.SchemaNode(colander.String(), name='body', widget=widgets.get_html_widget())) # Single file upload: #schema.add(colander.SchemaNode(deform.FileData(), name='attachment', widget=widgets.get_fileupload_widget(request))) # Sequence of file uploads: schema.add(colander.SchemaNode(colander.Sequence(), colander.SchemaNode(deform.FileData(), widget=widgets.get_fileupload_widget(request)), name='attachments', missing=[], default=[])) schema.add(colander.SchemaNode(colander.Boolean(), name='list_attachments', title="List attachments after body?", default=False, missing=False)) return schema
class Index: def __init__(self, ngram): self.tokenizer = Tokenizer("ma") self.docID = DocID() self.content = Content() self.ngram = ngram def tokenize(self, statement): #return self.tokenizer.split(statement, self.ngram) return self.tokenizer.split(statement) def append_doc(self, token, id, pos): return self.docID.set(token, id, pos) def set_content(self, statement): return self.content.set(statement) def append(self, statement): tokenized_str = self.tokenize(statement) content_id = self.set_content(statement) token_index = 0 for token in tokenized_str: self.append_doc(token, content_id, token_index) token_index += 1 def dump(self, dir): f_content_name = "content.pickle" f_docid_name = "docid.pickle" self.content.dump(f_content_name) self.docID.dump(f_docid_name) def load(self, dir): f_content_name = "content.pickle" f_docid_name = "docid.pickle" self.content.load(f_content_name) self.docID.dump(f_docid_name)
def dashboard_get_content(): gain_slider = SliderElement(key="antenna_gain", title="Antenna gain", width=1, range=(0, 50)) freq_slider = SliderElement(key="center_freq", title="Center Frequency", value=2.4, width=2, range=(2.38, 2.42), step=0.001) bin_slider = SliderElement(key="num_bins", title="Number of Bins", width=1, range=(100, 200)) win_len_slider = SliderElement(key="window_length", title="Detection windows", width=1, range=(1, 200)) vis1 = VisualisationElement(key="vis1", title="", default_type="fft", default_datatype="src_data") vis2 = VisualisationElement(key="vis2", title="", default_type="fft", default_datatype="rec_data") cnt = Content() cnt.add(gain_slider, position=(0, 0)) cnt.add(freq_slider, position=(1, 0)) cnt.add(bin_slider, position=(2, 0)) cnt.add(win_len_slider, position=(3, 0)) cnt.add(vis1, position=(0, 2)) cnt.add(vis2, position=(1, 2)) settings = ss.get_settings_object() return cnt
def __init__(self, cfg_name): self.lock = Lock() self.links = Queue() self.retry = Queue() self.fail = Queue() self.exist = set() self.complate = 0 self.running = 0 self.content = Content() # 配置信息 self.item_patterns = config[cfg_name]["item_patterns"] self.page_patterns = config[cfg_name]["page_patterns"] self.stop_patterns = config[cfg_name]["stop_patterns"] self.contents = config[cfg_name]["contents"] self.depth = config[cfg_name]["depth"] self.seeds = config[cfg_name]["seeds"] self.encoding = config[cfg_name]["encoding"] self.thread_number = config[cfg_name]["thread_number"] self.max_number = config[cfg_name]["max_number"]
class pastebinExtract(threading.Thread): def __init__(self,url,casperJSScript='pastebintext.js'): threading.Thread.__init__(self) self.url=url self.casperJSScript=casperJSScript self.content=Content() self.data=[] rua=Random_user_agent() self.ua=rua.rand() self.result=None def run(self): result=subprocess.Popen(['casperjs' ,self.casperJSScript,self.url,'\''+self.ua+'\''],stdout=PIPE) for ligne in result.stdout: record=ligne.strip() self.data.append(record.lower()) keywords=getattr(self.content,'keywords') for keyword in keywords: if self.content.search(keyword,str(self.data)): self.result={'url': self.url, 'data': self.data}
def _get_nonschema_mongo_save_document(self): doc = Content._get_nonschema_mongo_save_document(self) doc['_local_roles'] = self.get_local_roles() doc['_ordered_names'] = self.get_ordered_names() return doc
def _load_nonschema_attributes(self, **kwargs): Content._load_nonschema_attributes(self, **kwargs) _local_roles = kwargs.get('_local_roles') if _local_roles: self._local_roles = _local_roles _ordered_names = kwargs.get('_ordered_names') if _ordered_names is not None: self._ordered_names = _ordered_names
class Parse(object): def __init__(self): self.content = Content() self.db = MysqlControl() self._RST_RE = re.compile(r'\d+') self._ID_RE0 = re.compile(r'shopId:\s*(\d+)') self._ID_RE1 = re.compile(r'shopID:\s*(\d+)') self._NAME_RE = re.compile(r'cityCnName:\s*"(\S+)"') self._PHONE_RE = re.compile(r'电话:</span>\s+.+tel">(\d{3,4}.\d+)') self._REPLY_RE = re.compile(r'J-busi-reply') def parse_shop(self, info): basicInfo = list() soup = BeautifulSoup(info, "html5lib") shop_name_tag = soup.find(class_='shop-name') shop_brief_tag = soup.find(class_='brief-info') shop_addr_tag = soup.find(class_='expand-info address') addr = shop_addr_tag.find_all(class_='item')[0].string items = shop_brief_tag.find_all(class_='item') basicInfo.append(shop_name_tag.contents[0].string.strip()) basicInfo.append(shop_brief_tag.span['title']) for item in items[1:]: basicInfo.append(item.string[3:]) if len(basicInfo) == 3: basicInfo.extend(['-']*3) basicInfo.append(addr.strip()) shopId = self._ID_RE0.findall(info)[0] cityName = self._NAME_RE.findall(info)[0] phone_list = self._PHONE_RE.findall(info) if phone_list: phoneNum = phone_list[0] else: phoneNum = '-' res = {'shopId': shopId, 'cityName': cityName, 'phoneNum': phoneNum, 'basicInfo': basicInfo } self.db.add_shop(res) _Logger.info('shop {0}\'s basic info has saved'.format(shopId)) def parse_comment(self, comment): shopId = self._ID_RE1.findall(comment)[0] soup = BeautifulSoup(comment, "html5lib") div_tag = soup.find(class_='comment-list') content_tag = div_tag.find_all(class_='content') user_name_tag = div_tag.find_all(class_='name') def func(x, y): x.append(y) return x contents = map(func, content_tag, user_name_tag) for content in contents: reply_tag = content.find_all('div', self._REPLY_RE) time_tag = content.find_all(class_='time') text = content.find_all(class_='J_brief-cont')[0].contents[0].strip() if reply_tag: text += '\n' + reply_tag[0].find_all('p')[0].text.strip() userName = content.find(class_='name').contents[0].string rst = [] for x in content.find_all(class_='rst'): rst.extend(self._RST_RE.findall(x.contents[0])) time = time_tag[0].contents[0] if len(time) == 5: year = datetime.datetime.now().strftime('%y') time = year + '-' + time res = {'shopId': shopId, 'userName': userName, 'rst': rst, 'text': text, 'time': time } self.db.add_comment(res) _Logger.info('shop {0}\'s comments info has saved'.format(shopId)) def parse_all_info(self): for info, comment in self.content.get_all_contents(): self.parse_shop(info) self.parse_comment(comment)
def get_class_schema(cls, request=None): schema = Content.get_class_schema(request) schema.add(colander.SchemaNode(colander.Boolean(), name='_is_ordered', title="Enable child ordering?", default=False, missing=False, description="Enable this option if you need explicit control over ordering of child objects. Please avoid enabling this option on folders with a large number of children where sorting is more appropriate.")) return schema
def __init__(self, request, **kwargs): Content.__init__(self, request, **kwargs) self._collection_name = "content"
class Fetcher: # 初始化数据 def __init__(self, cfg_name): self.lock = Lock() self.links = Queue() self.retry = Queue() self.fail = Queue() self.exist = set() self.complate = 0 self.running = 0 self.content = Content() # 配置信息 self.item_patterns = config[cfg_name]["item_patterns"] self.page_patterns = config[cfg_name]["page_patterns"] self.stop_patterns = config[cfg_name]["stop_patterns"] self.contents = config[cfg_name]["contents"] self.depth = config[cfg_name]["depth"] self.seeds = config[cfg_name]["seeds"] self.encoding = config[cfg_name]["encoding"] self.thread_number = config[cfg_name]["thread_number"] self.max_number = config[cfg_name]["max_number"] # 解构的时候不必等待队列完成 def __del__(self): # self.links.join() pass # 启动线程 def start(self): for i in range(self.thread_number): t = Thread(target=self.run) t.setDaemon(True) t.start() for seed in self.seeds: self.push(seed) # 终止所有线程 def stop(self): pass # 增加任务数据 def push(self, link): if link["url"] not in self.exist: self.links.put(link) # 获得当前运行的线程数 def get_running_count(self): return self.running # 多线程主函数 def run(self): with self.lock: self.running += 1 while True: if self.complate >= self.max_number: break link = {} if self.retry.empty(): link = self.links.get() else: link = self.retry.get() try: response = self.openUrl(link["url"]) html = response.read() # 给网页重新编码,默认lxml只能处理utf-8 if self.encoding != "utf-8": html = html.decode(self.encoding, "ignore").encode("utf-8") self.extractContent(html, link["url"]) self.extractLinks(link["url"], html, link["depth"]) with self.lock: self.complate += 1 self.exist.add(link["url"]) except: if link.has_key("retry"): if link["retry"] >= 3: self.fail.put(link) else: link["retry"] += 1 self.retry.put(link) else: link["retry"] = 1 self.retry.put(link) print "Could not open %s" % link["url"] print "Error Info : %s " % sys.exc_info()[1] continue self.links.task_done() self.running -= 1 # 判断链接是否已经被爬取过 def isExist(self, url): if url in self.exist: return True return False # 判断链接是不是一个item def isItem(self, url): match = False for p in self.item_patterns: p = re.compile(p) if p.findall(url): match = True return match # 判断链接是不是page,是的话就不需要增加depth def isPage(self, url): match = False for p in self.page_patterns: p = re.compile(p) if p.findall(url): match = True return match # 判断链接是不是停止链接 def isStop(self, url): match = False for p in self.stop_patterns: p = re.compile(p) if p.findall(url): match = True return match # 打开url链接,返回数据流 def openUrl(self, url): headers = { "User-Agent": "Mozilla/5.0 \ (Macintosh; Intel Mac OS X 10_6_8) \ AppleWebKit/536.5 (KHTML, like Gecko) \ Chrome/19.0.1084.56 Safari/536.5" } req = urllib2.Request(url=url, data=None, headers=headers) c = urllib2.urlopen(req) return c # 从某个网页解析出所以符合条件的下一层链接 def extractLinks(self, referer, html, depth): soup = BeautifulSoup(html) tags = soup("a") # 从soup中获得所有的链接数据进行解析 for l in tags: if "href" in dict(l.attrs): url = urljoin(referer, l["href"]) url = url.split("#")[0] if not self.isExist(url) and depth < self.depth and self.isItem(url): # 如果当前链接是翻页链接,则不需要增加其depth if self.isPage(url): depth = depth + 1 link = {"url": "%s" % url, "parsed": False, "depth": depth} # 推送数据给任务队列 self.push(link) # 添加到已获取的链接集合中 self.exist.add(url) # 从某个网页解析出需要的内容 def extractContent(self, html, url): thread_name = current_thread().name t = time.strftime("%y-%m-%d %H:%M:%S", time.localtime()) Logger.write("[%s][%s][finish = %s][fail = %s] : %s" % (t, thread_name, self.complate, self.fail.qsize(), url)) parser = etree.XMLParser(ns_clean=True, recover=True) tree = etree.fromstring(html, parser) self.content.write(url, tree, self.contents) def printFinishLog(self): print "finish items: %s" % self.complate print "fail items : %s , list below :" % self.fail.qsize() while not self.fail.empty(): print self.fail.get()["url"] print ""
def get_compiled_task_list(self): task_list = self.get_task_list() content = Content(self.api) content.import_task_list(task_list) return content
# Methods for defining a character's stats and skills import json from content import Content import random game = Content().data engine = Content().engine the_stats = ["Evade","PhyDef","PhyAtk","MagAtk","MagDef", "Resistance","CarryStrength","Hit","Accuracy", "Craft","MaxHP"] CONSTRUCTS = engine.keys() def apply_level_mods(charac, rand=False): l = game["LISTS"]["Level"] mod_types = engine["Progression"]["Level"]["modifiers"] mods = [] levrange = charac["level"]+1 for lev in range(l[0], levrange): levobj = game[str(lev)] mods.extend(get_level_mods(levobj, charac)) # Take in a character object who has a certain level attribute # Iterate over level objects from zero to character level # Add to a master list of mods at each level, add mods to character charac = apply_mods(charac, mods, rand) return charac def get_level_mods(levelobj, charac): # Going to need a try/catch here for specific key errors
def _pre_delete(self): # If I'm being deleted, my kids are going down with me. for name in self.get_child_names(): self.delete_child(name) Content._pre_delete(self)
def _reset(self): self.content = None Content.trash(self.content_url) self._raw_content = None self._content_url = None self._messages = []
def __init__(self, ngram): self.tokenizer = Tokenizer("ma") self.docID = DocID() self.content = Content() self.ngram = ngram
def main(): """Create content""" import argparse parser = argparse.ArgumentParser( description="Creates content for music training.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) target_choices = ["puredjango", "simple_html", "android"] host_types = ["macports", "linux"] image_formats = ["png", "svg"] parser.add_argument("-t", "--target", help="Generate content for which format. "\ "Options: " + repr(target_choices) +".", choices=target_choices, default=target_choices[0]) parser.add_argument("-l", "--lecture", help="Generate content only for LECTURE "\ "(or list of semicolon separated lectures)", metavar="LECTURE", default=None) parser.add_argument("-n", "--only_new", action="store_true", help="Generate files only for missing stuff.") parser.add_argument("-H", "--host_type", choices=host_types, default=host_types[0], help="Host type for setting the paths to binaries. "\ "Options: " + repr(host_types) + ".") # parser.add_argument("-i", "--image_format", choices=image_formats, # default=image_formats[0], # help="Image output format. "\ # "Options: " + repr(host_types) + ".") parser.add_argument("-L", "--lilypond_path", metavar="path", help="Lilypond executable path", default=None) parser.add_argument('-v', '--verbose', dest="verbose", type=int, default=1, metavar='<int>', help="verbose level (default %(default)s)") args = parser.parse_args() if args.verbose >= 2: loglevel = logging.DEBUG elif args.verbose >= 1: loglevel = logging.INFO else: loglevel = logging.WARNING logging.basicConfig(format='%(module)s: %(message)s', level=loglevel) # Parse and expand the YAML files content = Content(lectures=set(args.lecture.split(";")) if args.lecture else None) content.expand() if args.target == "puredjango": target = PureDjangoTarget() elif args.target == "simple_html": target = SimpleHtmlTarget() elif args.target == "android": target = AndroidResourceTarget() target.include_images(content.fixed_images) sound_tasks, image_tasks = target.media_compile_tasklist( [e["lysrc"] for e in content.get_questions_and_choices()]) content.insert_filenames(sound_tasks, image_tasks, target.clean_fname) target.write(content.index) if args.host_type == "macports": binpath = "/opt/local/bin" elif args.host_type == "linux": binpath = "/usr/bin" timidity_path = binpath lilypond_path = binpath inkscape_path = binpath imagemagick_path = binpath if args.lilypond_path: lilypond_path = args.lilypond_path lcc = LilyCompiler(lilypond_path, imagemagick_path, inkscape_path, timidity_path, target.media_callbacks) lcc.compile(list(sound_tasks.values()) + list(image_tasks.values()), 4)
def contents_already_extracted_for(page): contents = Content.for_url(page.content_url) if len(contents) >= len(CONTENT_EXTRACTORS): return True, contents else: return False, contents
def create(): content = Content.from_json(request.json["cmd"]) return "OK"