Example #1
0
    def __init__(self, master):
        # Prevent the app window from being resized
        master.resizable(False, False)
        self.banner = Banner(master)
        self.content = Content(master)

        # KEYBOARD SHORTCUTS SECTION
        master.bind('<Control-n>', lambda e: self.content.add_task())

        # MENU BAR SECTION
        master.option_add('*tearOff', False)
        self.menubar = Menu(master)
        master.config(menu=self.menubar)

        # FILE MENU
        file = Menu(self.menubar)
        file.add_command(label='Test',
                         command=lambda: messagebox.showinfo(
                             title='Test', message='This is a test.'))
        file.add_command(label='Add Task',
                         command=lambda: self.content.add_task())
        file.entryconfig('Add Task', accelerator='CTRL+N')
        self.menubar.add_cascade(menu=file, label='File')

        # EDIT MENU
        edit = Menu(self.menubar)
        edit.add_command(label='Change Banner Color',
                         command=lambda: self.banner.change_banner_color())
        self.menubar.add_cascade(menu=edit, label='Edit')
 def setup_content_item_image_cat(self):
     tags = ["kitty", "kitten", "cat", "cute", "pet", "cutie", "animals"]
     content = Content("telegram", "@image_cat", tags, application="cats")
     image = ContentImage("image", "random", self.mockinfo.mockdirwfiles,
                          False, "json", self.mockinfo.historyfile)
     content.add_content_item(image)
     return content
Example #3
0
    def __init__(self,**kwargs):

        params  =  {
               'serviceID'     : 0x02,
               'userID'        : 0x00,
               'encodingType'  : 0x00,
               'length'        : 0x00,
               "userAccessLevel"  : 0x00,
               "userID"         : 0x00,
               "userNameLength" : 0x00,
               "userName"       : "",
               "systemStatus"        : 0x00,
               "systemStatusExtended": 0x00,
               "numberOfCounters"     :0x00,
               "counters"            : {}, # {'key' = counterID , counter'}
               "currentNetMaster"    : 0x00,
               'passcode'            : "000000",
            };

        for key in kwargs:
            if(key in params) != True:
                raise ValueError("@LogonContent : invalid parameter {0} ".format(key));
            else:
                params[key]  = kwargs[key];

        if(isinstance(params['passcode'], basestring)) != True:
           raise ValueError("@LogonContent: expecting passcode to be a string object.");
        Content.__init__(self,**params); 
Example #4
0
def sweep_get_content():
    sweep = SweepElement(key="sweep", title="Frequency sweep", default_datatype="src_data")

    cnt = Content()
    cnt.add(sweep, position=(0, 0))

    return cnt
    def __init__(self, **kwargs):

          attributes  =  {
               'panelNumber'           : 0x00, 
               'channel'               : ChannelType.MX_LOOP_TYPE,
               'loopNumber'            : 0xFF,  
               'pointCategory'         : PointCategoryType.padr_PCAT_REAL,
               'pointNumber'           : 0xFF, 
               'logicalPointNumber'    : 0xFD, 
               'logicalPointZone'      : 0xFE,
               'deviceCategory'        : DeviceCategoryType.ALL_DEVICE_TYPE,
               'group'                 : ByteCombinator().combineBytes(0x00, 0x01), #Two Bytes
               'outputPointStoreState' : 0x03,
               'reserved1'             : 0x00, 
               'reserved2'             : 0x00, 
               'multiAreaType'         : MultiAreaType.ALL,
               'areas'                 : Content.createSizeArray(29, 0xFF),
               'area'                  : 0xFF,
               'reserved'              : 0x01,
               'deviceType'            : devices.pdev_DEVICE_ID_ALL,
               'requestType'           : 0x01,
               'searchType'            : PointSearchType.ACTIVE_INPUTS
              };

          for key in kwargs:
              if ((key in attributes) != True):
                raise ValueError(key);
              attributes[key]=  kwargs[key];
          Content.__init__(self, **attributes);
Example #6
0
def interest():
    # get the user's tags
    userTags = Tag().user_tags(graph_db,
                               request.get_cookie(graphstoryUserAuthKey))
    # get the tags of user's friends
    tagsInNetwork = Tag().tags_in_network(
        graph_db, request.get_cookie(graphstoryUserAuthKey))

    # if the user's content was requested
    if request.query.get('userscontent') == "true":
        contents = Content().get_user_content_with_tag(
            graph_db, request.get_cookie(graphstoryUserAuthKey),
            request.query.get('tag'))
    # if the user's friends' content was requested
    else:
        contents = Content().get_following_content_with_tag(
            graph_db, request.get_cookie(graphstoryUserAuthKey),
            request.query.get('tag'))

    return template('public/templates/graphs/interest/index.html',
                    layout=applayout,
                    userTags=userTags,
                    tagsInNetwork=tagsInNetwork,
                    contents=contents,
                    title="Interest")
Example #7
0
    def test_copy_is_not_started_when_folder_is_empty(self, mock_terminal):
        mock_terminal.provide_initial_cwd.return_value = cwd
        mock_terminal.get_ls.side_effect = mock_return_values
        content = Content()

        content.copy_selected()

        self.assertEqual('', content.path_to_copy)
 def setup_content_item_text(self):
     content = Content("telegram", "@quotes_literature", "",
             application="quotes")
     format_rules = [{"\"%s":0},{"\"\n\n- %s":1}]
     text = ContentText("text", "random", "database",
                        self.mockinfo.mockdb, format_rules, "")
     content.add_content_item(text)
     return content
    def test_rename_is_called_with_correct_absolute_paths(self, mock_terminal):
        mock_terminal.provide_initial_cwd.return_value = cwd
        mock_terminal.get_ls.side_effect = mock_return_values
        content = Content()

        content.rename('aaaa', 'bbbb')

        mock_terminal.move.aassert_called_with(cwd + 'aaaa', cwd + 'bbbb')
Example #10
0
 def pub_workflow_transition(self, transition, save_children=True):
     """ Apply a workflow transition to this Folder, and (unless save_children==False)
     we also want to update "_view" fields of all children in both mongo and elastic by
     calling their save() method.
     """
     Content.pub_workflow_transition(self, transition)
     if save_children:
         save_recursively(self, include_self=False, set_modified=False)
Example #11
0
 def __init__(self, ngram, dir):
     self.docID = DocID()
     self.tokenizer = Tokenizer("ma")
     self.content = Content()
     self.ngram = ngram
     self.docID.load(dir + "docid.pickle")
     self.content.load(dir + "content.pickle")
     self.stopwords = self._load_stopwords(STOPWORDS_FILE)
Example #12
0
    def test_rename_is_not_called_when_folder_is_empty(self, mock_terminal):
        mock_terminal.provide_initial_cwd.return_value = cwd
        mock_terminal.get_ls.side_effect = mock_return_values
        content = Content()

        content.rename('a', 'b')

        mock_terminal.move.assert_not_called()
Example #13
0
    def test_delete_is_not_called_when_folder_is_empty(self, mock_terminal):
        mock_terminal.provide_initial_cwd.return_value = cwd
        mock_terminal.get_ls.side_effect = mock_return_values
        content = Content()

        content.delete_selected()

        mock_terminal.delete.assert_not_called()
Example #14
0
    def __init__(self, standard_screen):
        self.is_working = True
        self.standard_screen = standard_screen
        curses.curs_set(False)
        curses.init_pair(2, curses.COLOR_CYAN, curses.COLOR_BLACK)

        self.content = Content()
        self.input_keys = InputKeys()
        self.pane_manager = PaneManager(standard_screen)
Example #15
0
 def __init__(self):
     self.content = Content()
     self.db = MysqlControl()
     self._RST_RE = re.compile(r'\d+')
     self._ID_RE0 = re.compile(r'shopId:\s*(\d+)')
     self._ID_RE1 = re.compile(r'shopID:\s*(\d+)')
     self._NAME_RE = re.compile(r'cityCnName:\s*"(\S+)"')
     self._PHONE_RE = re.compile(r'电话:</span>\s+.+tel">(\d{3,4}.\d+)')
     self._REPLY_RE = re.compile(r'J-busi-reply')
    def test_delete_calls_delete_with_child_path(self, mock_terminal):
        mock_terminal.provide_initial_cwd.return_value = cwd
        mock_terminal.get_ls.side_effect = mock_return_values
        content = Content()
        content.main_pane_selected_line_i = 0

        content.delete_selected()

        mock_terminal.delete.assert_called_with(cwd + 'aaaa')
Example #17
0
def sweep_get_content():
    sweep = SweepElement(key="sweep",
                         title="Frequency sweep",
                         default_datatype="src_data")

    cnt = Content()
    cnt.add(sweep, position=(0, 0))

    return cnt
Example #18
0
 def __init__(self, url, casperJSScript='pastebintext.js'):
     threading.Thread.__init__(self)
     self.url = url
     self.casperJSScript = casperJSScript
     self.content = Content()
     self.data = []
     rua = Random_user_agent()
     self.ua = rua.rand()
     self.result = None
    def test_open_terminal_is_called_with_cwd(self, mock_terminal):
        mock_terminal.provide_initial_cwd.return_value = cwd
        mock_terminal.get_ls.side_effect = mock_return_values
        content = Content()
        content.main_pane_selected_line_i = 0

        content.open_new_terminal()

        mock_terminal.open_new_terminal.assert_called_with(content.cwd)
Example #20
0
 def setParameter(self, stype, values):
     if(stype =='counters'):
         if(self.validate(stype, values)):
             if(type(values)  == dict):
                 counter  =  self.getParameter(stype);
                 for key in values:
                    counter[key]  = values[key];
                 self._params[stype] =  counter;
     else:
         Content.setParameter(self, stype, values)
Example #21
0
    def __init__(self, parent, *args, **kwargs):
        tk.Frame.__init__(self, parent, *args, **kwargs)
        self.parent = parent
        self.parent.title('todo.txt')

        self.content = Content(self)
        self.content.grid(row=0, column=0, sticky='nsew')

        self.settings = Settings(self, self.content)
        self.parent.configure(menu=self.settings)
    def test_new_folder_calls_make_new_folder_with_name(self, mock_terminal):
        mock_terminal.provide_initial_cwd.return_value = cwd
        mock_terminal.get_ls.side_effect = mock_return_values
        content = Content()
        content.main_pane_selected_line_i = 0

        folder_name = 'new_folder_name'
        content.make_new_folder(folder_name)

        mock_terminal.make_new_folder.assert_called_with(cwd + folder_name)
    def test_copy_changes_path_to_copy_and_copy_removes_source(self, mock_terminal):
        mock_terminal.provide_initial_cwd.return_value = cwd
        mock_terminal.get_ls.side_effect = mock_return_values
        content = Content()
        content.main_pane_selected_line_i = 0

        content.copy_selected()

        self.assertEqual(cwd + 'aaaa', content.path_to_copy)
        self.assertFalse(content.copy_removes_source)
Example #24
0
def _list(content, args):
    """ List the available plugins. """
    list_content = Content()
    utils._populate_plugin_list(list_content, pfilter=args.string)
    if not len(list_content.plugin_list.plugin_list):
        print("No result found.")
        return content
    formatter = ListDisplay(list_content.plugin_list)
    verbosity = parsers._get_verbosity(args)
    list_content.display(formatter, verbose=verbosity)
    return content
    def __init__(self, **kwargs):

        attributes = {
            'clientID': 0x0000,  #Double bytes
        }

        for key in kwargs:
            if ((key in attributes) != True):
                raise ValueError(key)
            attributes[key] = kwargs[key]
        Content.__init__(self, **attributes)
Example #26
0
 def parse(self, site, url):
     """
     Extract content from a given page URL
     """
     bs = self.get_page(url)
     if bs is not None:
         title = self.safe_get(bs, site.titleTag)
         body = self.safe_get(bs, site.bodyTag)
         if title != '' and body != '':
             content = Content(url, title, body)
             content.print()
Example #27
0
def _list(content, args):
    """ List the available plugins. """
    list_content = Content()
    utils._populate_plugin_list(list_content, pfilter=args.string)
    if not len(list_content.plugin_list.plugin_list):
        print("No result found.")
        return content
    formatter = ListDisplay(list_content.plugin_list)
    verbosity = parsers._get_verbosity(args)
    list_content.display(formatter, verbose=verbosity)
    return content
Example #28
0
    def handle_select_item(self, item, args):
        content_data = self.game.datastore.contents[item.content]
        content = Content(self, content_data.title, content_data.text)
        self.game.datastore.datamodel.backpack[
            self.game.datastore.datamodel.level - 1].add(item.content)
        item.icon_item.set_visible(True)
        content.start()
        self.render()
        self.item_found_sound.play()

        # Save data
        self.game.datastore.changed_data = True
Example #29
0
    def __init__(self, **kwargs):
        params = {
            "resynchronisationData": 0x00,
            "avaliable": 0x00,
            "resynchronisationSent": 0x00
        }

        for key in kwargs:
            if key in params:
                params[key] = kwargs[key]

        Content.__init__(self, **params)
Example #30
0
def main():

    print("Running the configurator")
    # required for travis tests
    if len(sys.argv) > 2 and sys.argv[-2] == 'scripts/configurator_tests/':
        sys.argv = [sys.argv[:-2]]

    args = parsers._config_arg_parser()
    if args.error:
        utils.error_level = 1

    print("Starting Savu Config tool (please wait for prompt)")

    _reduce_logging_level()

    content = Content(level="all" if args.disp_all else 'user')

    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        content.failed = utils.populate_plugins(error_mode=args.error,
                                                examples=args.examples)

    comp = Completer(commands=commands, plugin_list=pu.plugins)
    utils._set_readline(comp.complete)

    # if file flag is passed then open it here
    if args.file:
        commands['open'](content, args.file)

    print("\n*** Press Enter for a list of available commands. ***\n")

    utils.load_history_file(utils.histfile)

    while True:
        try:
            in_list = raw_input(">>> ").strip().split(' ', 1)
        except KeyboardInterrupt:
            print()
            continue

        command, arg = in_list if len(in_list) == 2 else in_list + ['']
        command = command if command else 'help'
        if command not in commands:
            print("I'm sorry, that's not a command I recognise. Press Enter "
                  "for a list of available commands.")
        else:
            content = commands[command](content, arg)

        if content.is_finished():
            break

    print("Thanks for using the application")
Example #31
0
def prepare_email_content(email_address, subject):
    #construct Receiver, Content and Attachment Class
    r = Receiver()
    c = Content()
    a = Attachment()

    msg = EmailMessage()
    msg['Subject'] = subject
    msg['To'] = ', '.join(r.get_receiver())
    msg['From'] = email_address
    msg.set_content(c.get_content())
    set_attachment(msg, a.get_attachments())
    return msg
Example #32
0
File: tP.py Project: imcmy/tP
 def __init__(self, file_path):
     self.__init_win()
     content = Content(file_path)
     self.image_path = os.path.dirname(file_path)
     self.presentation_content = content.get_content_list()
     self.currentpath = os.getcwd()
     try:
         self.show_str("tP by jinsheng, modified by imcmy. press 'j' to continue", 0, False)
         self.start()
     except KeyboardInterrupt:
         pass
     self.__reset_background()
     curses.endwin()
Example #33
0
File: tP.py Project: scusjs/tP
 def __init__(self, file_path):
     self.__init_win()
     content = Content(file_path)
     self.image_path = os.path.dirname(file_path)
     self.presentation_content = content.get_content_list()
     self.currentpath = os.getcwd()
     try:
         self.show_str("tP by jinsheng, press 'j' to continue", 0, False)
         self.start()
     except KeyboardInterrupt:
         pass
     self.__reset_background()
     curses.endwin()
Example #34
0
 def handle_select_item(self, item, args):
     content_data = self.game.datastore.contents[item.content]
     item.set_rollover_image(None)
     item.remove_event_handler(ItemEvent.CLICK, self.handle_select_item)
     content = Content(self, content_data.title, content_data.text)
     self.game.datastore.datamodel.backpack[self.game.datastore.datamodel.level - 1].add(item.content)
     item.icon_item.set_visible(True)
     content.start()
     self.render()
     self.item_found_sound.play()
     
     # Save data
     self.game.datastore.changed_data = True
Example #35
0
File: tP.py Project: secondwtq/tP
 def __init__(self, file_path):
     content = Content(file_path)
     self.image_path = os.path.dirname(file_path)
     self.presentation_content = content.get_content_list()
     self.currentpath = os.getcwd()
     self.screen = curses.initscr()
     self.screen.keypad(True)
     curses.noecho()
     curses.cbreak()
     self.height, self.width = self.screen.getmaxyx()
     self.show_str("tP by jinsheng, press 'j' to continue", 0, False)
     self.start()
     self.__reset_background()
     curses.endwin()
Example #36
0
def main():

    # required for travis tests
    if len(sys.argv) > 2 and sys.argv[-2] == 'scripts/configurator_tests/':
        sys.argv = [sys.argv[:-2]]

    args = parsers._config_arg_parser()
    if args.error:
        utils.error_level = 1

    print("Starting Savu Config tool (please wait for prompt)")

    utils.populate_plugins()
    comp = Completer(commands=commands, plugin_list=pu.plugins)
    utils._set_readline(comp.complete)

    content = Content(level="all" if args.disp_all else 'user')

    # if file flag is passed then open it here
    if args.file:
        commands['open'](content, args.file)

    print("\n*** Press Enter for a list of available commands. ***\n")

    while True:
        try:
            in_list = raw_input(">>> ").strip().split(' ', 1)
        except KeyboardInterrupt:
            print()
            continue

        command, arg = in_list if len(in_list) is 2 else in_list+['']
        command = command if command else 'help'
        if command not in commands:
            print("I'm sorry, that's not a command I recognise. Press Enter "
                  "for a list of available commands.")
        else:
            content = commands[command](content, arg)

        if content.is_finished():
            break

        # write the history to the history file
        utils.readline.write_history_file(utils.histfile)

    print("Thanks for using the application")
Example #37
0
 def __init__(self, ngram, dir):
   self.docID = DocID()
   self.tokenizer = Tokenizer("ma")
   self.content = Content()
   self.ngram = ngram
   self.docID.load(dir + "docid.pickle")
   self.content.load(dir + "content.pickle")
   self.stopwords = self._load_stopwords(STOPWORDS_FILE)
Example #38
0
 def __init__(self):
     self.content = Content()
     self.db = MysqlControl()
     self._RST_RE = re.compile(r'\d+')
     self._ID_RE0 = re.compile(r'shopId:\s*(\d+)')
     self._ID_RE1 = re.compile(r'shopID:\s*(\d+)')
     self._NAME_RE = re.compile(r'cityCnName:\s*"(\S+)"')
     self._PHONE_RE = re.compile(r'电话:</span>\s+.+tel">(\d{3,4}.\d+)')
     self._REPLY_RE = re.compile(r'J-busi-reply')
 def __init__(self,url,casperJSScript='pastebintext.js'):
     threading.Thread.__init__(self)
     self.url=url
     self.casperJSScript=casperJSScript
     self.content=Content()
     self.data=[]
     rua=Random_user_agent()
     self.ua=rua.rand()
     self.result=None
Example #40
0
def task_extract_content(extractor, page_key):
	page = Page.get(page_key)
	if not page.pending or Content.already_fetched(page.content_url, extractor):
		info("skipping content extraction [%s] for page %s, as it is not pending or already fetched" %
			(extractor, page_key))
		return
	try:
		get_content_extractor(extractor).extract(page)
	except Exception, e:
		error("content extraction %s for page:%s failed with error <%s: %s>" %
			(extractor, page_key, type(e), e))
		raise
Example #41
0
 def search(self, topic, site):
     """
     Searches a given website for a given topic and records all pages found
     """
     bsObj = self.getPage(site.searchUrl+topic)
     searchResults = bsObj.select(site.resultListing)
     for result in searchResults:
         url = result.select(site.resultUrl)[0].attrs["href"]
         #Check to see whether it's a relative or an absolute URL
         if(site.absoluteUrl):
             pageObj = self.getPage(url)
         else:
             pageObj = self.getPage(site.url+url)
         if pageObj is None:
             print("Something was wrong with that page or URL. Skipping!")
             return
         title = self.safeGet(pageObj, site.pageTitle)
         body = self.safeGet(pageObj, site.pageBody)
         if title != "" and body != "":
             content = Content(topic, title, body, url)
             content.print()
Example #42
0
    def get_class_schema(cls, request=None):
        schema = Content.get_class_schema(request)
        #schema.add(colander.SchemaNode(colander_types.DateUS(), name='dateline', default=today_for_request_tz(request)))
        schema.add(colander.SchemaNode(colander_types.DateTimeUS(get_timezone_for_request(request)), name='dateline', default=utcnow()))
        schema.add(colander.SchemaNode(colander.String(), name='body', widget=widgets.get_html_widget()))
        # Single file upload:
        #schema.add(colander.SchemaNode(deform.FileData(), name='attachment', widget=widgets.get_fileupload_widget(request)))

        # Sequence of file uploads:
        schema.add(colander.SchemaNode(colander.Sequence(), colander.SchemaNode(deform.FileData(), widget=widgets.get_fileupload_widget(request)), name='attachments', missing=[], default=[]))
        schema.add(colander.SchemaNode(colander.Boolean(), name='list_attachments', title="List attachments after body?", default=False, missing=False))

        return schema
Example #43
0
class Index:
  def __init__(self, ngram):
    self.tokenizer = Tokenizer("ma")
    self.docID = DocID()
    self.content = Content()
    self.ngram = ngram

  def tokenize(self, statement):
    #return self.tokenizer.split(statement, self.ngram)
    return self.tokenizer.split(statement)

  def append_doc(self, token, id, pos):
    return self.docID.set(token, id, pos)

  def set_content(self, statement):
    return self.content.set(statement)

  def append(self, statement):
    tokenized_str = self.tokenize(statement)
    content_id = self.set_content(statement)

    token_index = 0

    for token in tokenized_str:
      self.append_doc(token, content_id, token_index)
      token_index += 1 

  def dump(self, dir):
    f_content_name = "content.pickle"
    f_docid_name = "docid.pickle"
    self.content.dump(f_content_name)
    self.docID.dump(f_docid_name)

  def load(self, dir):
    f_content_name = "content.pickle"
    f_docid_name = "docid.pickle"

    self.content.load(f_content_name)
    self.docID.dump(f_docid_name)
Example #44
0
def dashboard_get_content():
    gain_slider = SliderElement(key="antenna_gain", title="Antenna gain",
                                width=1, range=(0, 50))
    freq_slider = SliderElement(key="center_freq", title="Center Frequency",
                                value=2.4, width=2, range=(2.38, 2.42), step=0.001)
    bin_slider = SliderElement(key="num_bins", title="Number of Bins", width=1, range=(100, 200))
    win_len_slider = SliderElement(key="window_length", title="Detection windows", width=1, range=(1, 200))
    vis1 = VisualisationElement(key="vis1", title="", default_type="fft", default_datatype="src_data")
    vis2 = VisualisationElement(key="vis2", title="", default_type="fft", default_datatype="rec_data")

    cnt = Content()
    cnt.add(gain_slider, position=(0, 0))
    cnt.add(freq_slider, position=(1, 0))
    cnt.add(bin_slider, position=(2, 0))
    cnt.add(win_len_slider, position=(3, 0))
    cnt.add(vis1, position=(0, 2))
    cnt.add(vis2, position=(1, 2))

    settings = ss.get_settings_object()

    return cnt
Example #45
0
    def __init__(self, cfg_name):
        self.lock = Lock()
        self.links = Queue()
        self.retry = Queue()
        self.fail = Queue()
        self.exist = set()
        self.complate = 0
        self.running = 0
        self.content = Content()

        # 配置信息
        self.item_patterns = config[cfg_name]["item_patterns"]
        self.page_patterns = config[cfg_name]["page_patterns"]
        self.stop_patterns = config[cfg_name]["stop_patterns"]
        self.contents = config[cfg_name]["contents"]
        self.depth = config[cfg_name]["depth"]
        self.seeds = config[cfg_name]["seeds"]
        self.encoding = config[cfg_name]["encoding"]
        self.thread_number = config[cfg_name]["thread_number"]
        self.max_number = config[cfg_name]["max_number"]
class pastebinExtract(threading.Thread):
    def __init__(self,url,casperJSScript='pastebintext.js'):
        threading.Thread.__init__(self)
        self.url=url
        self.casperJSScript=casperJSScript
        self.content=Content()
        self.data=[]
        rua=Random_user_agent()
        self.ua=rua.rand()
        self.result=None
   
    def run(self):
        result=subprocess.Popen(['casperjs' ,self.casperJSScript,self.url,'\''+self.ua+'\''],stdout=PIPE)
        for ligne in result.stdout:
                record=ligne.strip()                
                self.data.append(record.lower())
                
        keywords=getattr(self.content,'keywords')
        for keyword in keywords:
            if self.content.search(keyword,str(self.data)):
                self.result={'url': self.url, 'data': self.data}
Example #47
0
 def _get_nonschema_mongo_save_document(self):
     doc = Content._get_nonschema_mongo_save_document(self)
     doc['_local_roles'] = self.get_local_roles()
     doc['_ordered_names'] = self.get_ordered_names()
     return doc
Example #48
0
 def _load_nonschema_attributes(self, **kwargs):
     Content._load_nonschema_attributes(self, **kwargs)
     _local_roles = kwargs.get('_local_roles')
     if _local_roles: self._local_roles = _local_roles
     _ordered_names = kwargs.get('_ordered_names')
     if _ordered_names is not None: self._ordered_names = _ordered_names
Example #49
0
class Parse(object):

    def __init__(self):
        self.content = Content()
        self.db = MysqlControl()
        self._RST_RE = re.compile(r'\d+')
        self._ID_RE0 = re.compile(r'shopId:\s*(\d+)')
        self._ID_RE1 = re.compile(r'shopID:\s*(\d+)')
        self._NAME_RE = re.compile(r'cityCnName:\s*"(\S+)"')
        self._PHONE_RE = re.compile(r'电话:</span>\s+.+tel">(\d{3,4}.\d+)')
        self._REPLY_RE = re.compile(r'J-busi-reply')

    def parse_shop(self, info):
        basicInfo = list()
        soup = BeautifulSoup(info, "html5lib")
        shop_name_tag = soup.find(class_='shop-name')
        shop_brief_tag = soup.find(class_='brief-info')
        shop_addr_tag = soup.find(class_='expand-info address')
        addr = shop_addr_tag.find_all(class_='item')[0].string
        items = shop_brief_tag.find_all(class_='item')
        basicInfo.append(shop_name_tag.contents[0].string.strip())
        basicInfo.append(shop_brief_tag.span['title'])
        for item in items[1:]:
            basicInfo.append(item.string[3:])
        if len(basicInfo) == 3:
            basicInfo.extend(['-']*3)
        basicInfo.append(addr.strip())
        shopId = self._ID_RE0.findall(info)[0]
        cityName = self._NAME_RE.findall(info)[0]
        phone_list = self._PHONE_RE.findall(info)
        if phone_list:
            phoneNum = phone_list[0]
        else:
            phoneNum = '-'
        res = {'shopId': shopId,
               'cityName': cityName,
               'phoneNum': phoneNum,
               'basicInfo': basicInfo
        }
        self.db.add_shop(res)
        _Logger.info('shop {0}\'s basic info has saved'.format(shopId))

    def parse_comment(self, comment):
        shopId = self._ID_RE1.findall(comment)[0]
        soup = BeautifulSoup(comment, "html5lib")
        div_tag = soup.find(class_='comment-list')
        content_tag = div_tag.find_all(class_='content')
        user_name_tag = div_tag.find_all(class_='name')
        def func(x, y):
            x.append(y)
            return x

        contents = map(func, content_tag, user_name_tag)
        for content in contents:
            reply_tag = content.find_all('div', self._REPLY_RE)
            time_tag = content.find_all(class_='time')
            text = content.find_all(class_='J_brief-cont')[0].contents[0].strip()
            if reply_tag:
                text += '\n' + reply_tag[0].find_all('p')[0].text.strip()
            userName = content.find(class_='name').contents[0].string
            rst = []
            for x in content.find_all(class_='rst'):
                rst.extend(self._RST_RE.findall(x.contents[0]))
            time = time_tag[0].contents[0]
            if len(time) == 5:
                year = datetime.datetime.now().strftime('%y')
                time = year + '-' + time
            res = {'shopId': shopId,
                   'userName': userName,
                   'rst': rst,
                   'text': text,
                   'time': time
            }
            self.db.add_comment(res)
        _Logger.info('shop {0}\'s comments info has saved'.format(shopId))

    def parse_all_info(self):
        for info, comment in self.content.get_all_contents():
            self.parse_shop(info)
            self.parse_comment(comment)
Example #50
0
    def get_class_schema(cls, request=None):
        schema = Content.get_class_schema(request)
        schema.add(colander.SchemaNode(colander.Boolean(), name='_is_ordered', title="Enable child ordering?", default=False, missing=False, description="Enable this option if you need explicit control over ordering of child objects.  Please avoid enabling this option on folders with a large number of children where sorting is more appropriate."))

        return schema
Example #51
0
 def __init__(self, request, **kwargs):
     Content.__init__(self, request, **kwargs)
     self._collection_name = "content"
Example #52
0
class Fetcher:
    # 初始化数据
    def __init__(self, cfg_name):
        self.lock = Lock()
        self.links = Queue()
        self.retry = Queue()
        self.fail = Queue()
        self.exist = set()
        self.complate = 0
        self.running = 0
        self.content = Content()

        # 配置信息
        self.item_patterns = config[cfg_name]["item_patterns"]
        self.page_patterns = config[cfg_name]["page_patterns"]
        self.stop_patterns = config[cfg_name]["stop_patterns"]
        self.contents = config[cfg_name]["contents"]
        self.depth = config[cfg_name]["depth"]
        self.seeds = config[cfg_name]["seeds"]
        self.encoding = config[cfg_name]["encoding"]
        self.thread_number = config[cfg_name]["thread_number"]
        self.max_number = config[cfg_name]["max_number"]

    # 解构的时候不必等待队列完成
    def __del__(self):
        # self.links.join()
        pass

    # 启动线程
    def start(self):
        for i in range(self.thread_number):
            t = Thread(target=self.run)
            t.setDaemon(True)
            t.start()

        for seed in self.seeds:
            self.push(seed)

    # 终止所有线程
    def stop(self):
        pass

    # 增加任务数据
    def push(self, link):
        if link["url"] not in self.exist:
            self.links.put(link)

    # 获得当前运行的线程数
    def get_running_count(self):
        return self.running

    # 多线程主函数
    def run(self):
        with self.lock:
            self.running += 1
        while True:
            if self.complate >= self.max_number:
                break
            link = {}
            if self.retry.empty():
                link = self.links.get()
            else:
                link = self.retry.get()

            try:
                response = self.openUrl(link["url"])
                html = response.read()
                # 给网页重新编码,默认lxml只能处理utf-8
                if self.encoding != "utf-8":
                    html = html.decode(self.encoding, "ignore").encode("utf-8")
                self.extractContent(html, link["url"])
                self.extractLinks(link["url"], html, link["depth"])
                with self.lock:
                    self.complate += 1
                    self.exist.add(link["url"])
            except:
                if link.has_key("retry"):
                    if link["retry"] >= 3:
                        self.fail.put(link)
                    else:
                        link["retry"] += 1
                        self.retry.put(link)
                else:
                    link["retry"] = 1
                    self.retry.put(link)
                print "Could not open %s" % link["url"]
                print "Error Info : %s " % sys.exc_info()[1]
                continue

            self.links.task_done()
        self.running -= 1

    # 判断链接是否已经被爬取过
    def isExist(self, url):
        if url in self.exist:
            return True
        return False

    # 判断链接是不是一个item
    def isItem(self, url):
        match = False
        for p in self.item_patterns:
            p = re.compile(p)
            if p.findall(url):
                match = True
        return match

    # 判断链接是不是page,是的话就不需要增加depth
    def isPage(self, url):
        match = False
        for p in self.page_patterns:
            p = re.compile(p)
            if p.findall(url):
                match = True
        return match

    # 判断链接是不是停止链接
    def isStop(self, url):
        match = False
        for p in self.stop_patterns:
            p = re.compile(p)
            if p.findall(url):
                match = True
        return match

    # 打开url链接,返回数据流
    def openUrl(self, url):
        headers = {
            "User-Agent": "Mozilla/5.0 \
                (Macintosh; Intel Mac OS X 10_6_8) \
                AppleWebKit/536.5 (KHTML, like Gecko) \
                Chrome/19.0.1084.56 Safari/536.5"
        }
        req = urllib2.Request(url=url, data=None, headers=headers)
        c = urllib2.urlopen(req)
        return c

    # 从某个网页解析出所以符合条件的下一层链接
    def extractLinks(self, referer, html, depth):
        soup = BeautifulSoup(html)
        tags = soup("a")
        # 从soup中获得所有的链接数据进行解析
        for l in tags:
            if "href" in dict(l.attrs):
                url = urljoin(referer, l["href"])
                url = url.split("#")[0]
                if not self.isExist(url) and depth < self.depth and self.isItem(url):
                    # 如果当前链接是翻页链接,则不需要增加其depth
                    if self.isPage(url):
                        depth = depth + 1
                    link = {"url": "%s" % url, "parsed": False, "depth": depth}
                    # 推送数据给任务队列
                    self.push(link)
                    # 添加到已获取的链接集合中
                    self.exist.add(url)

    # 从某个网页解析出需要的内容
    def extractContent(self, html, url):
        thread_name = current_thread().name
        t = time.strftime("%y-%m-%d %H:%M:%S", time.localtime())
        Logger.write("[%s][%s][finish = %s][fail = %s] : %s" % (t, thread_name, self.complate, self.fail.qsize(), url))
        parser = etree.XMLParser(ns_clean=True, recover=True)
        tree = etree.fromstring(html, parser)
        self.content.write(url, tree, self.contents)

    def printFinishLog(self):
        print "finish items: %s" % self.complate
        print "fail items : %s , list below :" % self.fail.qsize()
        while not self.fail.empty():
            print self.fail.get()["url"]
        print ""
Example #53
0
    def get_compiled_task_list(self):
        task_list = self.get_task_list()

        content = Content(self.api)
        content.import_task_list(task_list)
        return content
Example #54
0
# Methods for defining a character's stats and skills

import json
from content import Content
import random

game = Content().data
engine = Content().engine

the_stats = ["Evade","PhyDef","PhyAtk","MagAtk","MagDef",
			"Resistance","CarryStrength","Hit","Accuracy",
			"Craft","MaxHP"]

CONSTRUCTS = engine.keys()

def apply_level_mods(charac, rand=False):
	l = game["LISTS"]["Level"]
	mod_types = engine["Progression"]["Level"]["modifiers"]
	mods = []
	levrange = charac["level"]+1
	for lev in range(l[0], levrange):
		levobj = game[str(lev)]
		mods.extend(get_level_mods(levobj, charac))
	# Take in a character object who has a certain level attribute
	# Iterate over level objects from zero to character level
	# Add to a master list of mods at each level, add mods to character
	charac = apply_mods(charac, mods, rand)
	return charac

def get_level_mods(levelobj, charac):
	# Going to need a try/catch here for specific key errors
Example #55
0
 def _pre_delete(self):
     # If I'm being deleted, my kids are going down with me.
     for name in self.get_child_names():
         self.delete_child(name)
     Content._pre_delete(self)
Example #56
0
	def _reset(self):
		self.content = None
		Content.trash(self.content_url)
		self._raw_content = None
		self._content_url = None
		self._messages = []
Example #57
0
 def __init__(self, ngram):
   self.tokenizer = Tokenizer("ma")
   self.docID = DocID()
   self.content = Content()
   self.ngram = ngram
Example #58
0
def main():
    """Create content"""
    import argparse

    parser = argparse.ArgumentParser(
        description="Creates content for music training.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)


    target_choices = ["puredjango", "simple_html", "android"]
    host_types = ["macports", "linux"]
    image_formats = ["png", "svg"]

    parser.add_argument("-t", "--target",
                        help="Generate content for which format. "\
                          "Options: " + repr(target_choices) +".",
                        choices=target_choices,
                        default=target_choices[0])
    parser.add_argument("-l", "--lecture",
                        help="Generate content only for LECTURE "\
                        "(or list of semicolon separated lectures)",
                        metavar="LECTURE", default=None)
    parser.add_argument("-n", "--only_new", action="store_true",
                        help="Generate files only for missing stuff.")
    parser.add_argument("-H", "--host_type", choices=host_types,
                        default=host_types[0],
                        help="Host type for setting the paths to binaries. "\
                          "Options: " + repr(host_types) + ".")
   # parser.add_argument("-i", "--image_format", choices=image_formats,
   #                     default=image_formats[0],
   #                     help="Image output format. "\
   #                       "Options: " + repr(host_types) + ".")
    parser.add_argument("-L", "--lilypond_path", metavar="path",
                        help="Lilypond executable path",
                        default=None)
    parser.add_argument('-v', '--verbose', dest="verbose", type=int,
                        default=1, metavar='<int>',
                        help="verbose level (default %(default)s)")

    args = parser.parse_args()

    if args.verbose >= 2:
        loglevel = logging.DEBUG
    elif args.verbose >= 1:
        loglevel = logging.INFO
    else:
        loglevel = logging.WARNING
    logging.basicConfig(format='%(module)s: %(message)s',
                        level=loglevel)

    # Parse and expand the YAML files
    content = Content(lectures=set(args.lecture.split(";")) if args.lecture else None)
    content.expand()

    if args.target == "puredjango":
        target = PureDjangoTarget()
    elif args.target == "simple_html":
        target = SimpleHtmlTarget()
    elif args.target == "android":
        target = AndroidResourceTarget()

    target.include_images(content.fixed_images)
    sound_tasks, image_tasks = target.media_compile_tasklist(
        [e["lysrc"] for e in content.get_questions_and_choices()])
    content.insert_filenames(sound_tasks, image_tasks, target.clean_fname)
    target.write(content.index)

    if args.host_type == "macports":
        binpath = "/opt/local/bin"
    elif args.host_type == "linux":
        binpath = "/usr/bin"

    timidity_path = binpath
    lilypond_path = binpath
    inkscape_path = binpath
    imagemagick_path = binpath
    if args.lilypond_path:
        lilypond_path = args.lilypond_path

    lcc = LilyCompiler(lilypond_path, imagemagick_path, inkscape_path, timidity_path,
                       target.media_callbacks)
    lcc.compile(list(sound_tasks.values()) + list(image_tasks.values()), 4)
Example #59
0
def contents_already_extracted_for(page):
	contents = Content.for_url(page.content_url)
	if len(contents) >= len(CONTENT_EXTRACTORS):
		return True, contents
	else:
		return False, contents
Example #60
0
def create():
    content = Content.from_json(request.json["cmd"])
    return "OK"