示例#1
0
    def test_getBsObjectContent(self):
        ''' Testing if getBsObject returns the right content. '''
        actual = (helper.getBsObject(self.url)).prettify()
        expected_same = helper.readFile('pythonscraping_example.html')
        expected_diff = helper.readFile('pythonscraping_wrong_example.html')

        self.assertEqual(actual, expected_same)
        self.assertNotEqual(actual, expected_diff)
示例#2
0
    def test_readFileCorrect(self):
        ''' Tests if the file is read accurately. '''
        actual_simp = helper.readFile('testfile')
        actual_utf8 = helper.readFile('testfile_strange')
        expected_simp = 'Testing if this file gets modified.'
        expected_utf8 = 'Testing with strange input √√«ß‹∫‹∑ ß∆ñ∆œ∑ß...'

        self.assertEqual(actual_simp, expected_simp)
        self.assertEqual(actual_utf8, expected_utf8)
示例#3
0
    def test_removeComments(self):
        ''' Tests if the outputfile has removed comments as it should have. '''
        orginalfile = helper.readFile('removecommentsfile')
        correctfile = helper.readFile('correct_removecomments')

        outputfile = helper.removeComments(orginalfile)

        actual = hashlib.md5(str(outputfile).encode()).hexdigest()
        expected = hashlib.md5(str(correctfile).encode()).hexdigest()

        self.assertEqual(actual, expected)
 def create_jsc_folder(self):
     jsc_folder = self.target_directory + '/' + 'jsc'
     helper.create_directory(jsc_folder)
     helper.create_directory(jsc_folder + '/cocos2d_js')
     helper.copytree(sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/jsc/cocos2d_js", jsc_folder + '/cocos2d_js')
     # create complier_config.json
     complier_config_file = sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/jsc/compiler_config.json"
     complier_config_content = helper.readFile(complier_config_file)
     helper.writeFile(jsc_folder + "/complier_config.json", complier_config_content)
     # create generate_jsc.py
     generate_jsc_file = sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/jsc/generate_jsc.py"
     generate_jsc_content = helper.readFile(generate_jsc_file)
     generate_jsc_content = generate_jsc_content.replace("%BINDING_JS_FOLDER%", sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/jsc/bindings")
     generate_jsc_content = generate_jsc_content.replace("%BIN_EXE_PATH%", sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/jsc/bin/jsbcc")
     helper.writeFile(jsc_folder + "/generate_jsc.py", generate_jsc_content)
def dealWithFile(filepath):
	#get filename to mkdir
	base_name = os.path.basename(filepath)
	base_name = os.path.splitext(base_name)[0]
	parent_path = os.path.abspath(os.path.join(output_path,base_name))
	if(not os.path.exists(parent_path)):
		os.mkdir(parent_path)
	
	readfp = helper.readFile(filepath).splitlines()
	
	while len(readfp)>0:
		line_text = readfp.pop(0);
		#first match className
		result = re.match(r'.*@module (\w+)', line_text)
		if result:
			className = result.group(1)
			dealWithModule(className, readfp, parent_path)
		else:
			#match function
			result = re.match(r'.*@function .* (\w+)', line_text)
			if result:
				funcName = result.group(1)
				#too many function named create,deal it single
				if funcName == "create":
					dealWithCreate(className, readfp, parent_path)
				elif funcName != className:         #ignore constructor
					dealWithFunction(funcName, readfp, parent_path, className)
 def create_ant_folder(self):
     ant_folder = self.target_directory + '/' + 'ant'
     helper.create_directory(ant_folder)
     # create build.xml
     build_file = sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/ant/build.xml"
     build_content = helper.readFile(build_file)
     build_content = build_content.replace("%MODE%", helper.loadSettings("Cocos2dJSDev").get("ant").get("mode"))
     build_content = build_content.replace("%DEBUG%", helper.loadSettings("Cocos2dJSDev").get("ant").get("debug"))
     build_content = build_content.replace("%OUTPUT.js%", helper.loadSettings("Cocos2dJSDev").get("ant").get("output"))
     build_content = build_content.replace("%COMPLIER_JAR_PATH%", sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/ant")
     build_content = build_content.replace("%COCOS2D_ROOT_PATH%", helper.loadSettings("Cocos2dJSDev").get("cocos2d_html5_root"))
     helper.writeFile(ant_folder + "/build.xml", build_content)
     # create cocos2d_extern.js
     cocos2d_externs_file = sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/ant/cocos2d_externs.js"
     cocos2d_externs_content = helper.readFile(cocos2d_externs_file)
     helper.writeFile(ant_folder + "/cocos2d_externs.js", cocos2d_externs_content)
示例#7
0
def init():
    global TEMP_PATH
    TEMP_PATH=sublime.packages_path()+"/User/QuickXDev.cache"
    global USER_DEFINITION_LIST
    path=os.path.join(TEMP_PATH,"user_definition.json")
    if os.path.exists(path):
        USER_DEFINITION_LIST=json.loads(helper.readFile(path))
示例#8
0
def init():
    global TEMP_PATH
    TEMP_PATH=sublime.packages_path()+"/User/quick-comminuty-dev.cache"
    global DEFINITION_LIST
    DEFINITION_LIST=json.loads(definition.data)
    global USER_DEFINITION_LIST
    path=os.path.join(TEMP_PATH,"user_definition.json")
    if os.path.exists(path):
        USER_DEFINITION_LIST=json.loads(helper.readFile(path))
 def run(self, files):
     self.target_file = files[0]
     resource_content = helper.readFile(self.target_file)
     resources = re.findall(r"\s*{src:\s*(.*)}", resource_content)
     # print param_list
     del RESOURCE_LIST[:]
     for resource in resources:
         RESOURCE_LIST.append(resource)
     sublime.message_dialog("Update Resource List Done!")
示例#10
0
def init():
    global TEMP_PATH,DEFINITION_LIST,USER_DEFINITION_LIST
    TEMP_PATH=sublime.packages_path()+"/User/lufylegendDev.cache"
    lufylegend_root = checkRoot(False)
    if lufylegend_root:
        initDefinition(lufylegend_root)
    path=os.path.join(TEMP_PATH,"user_definition.json")
    if os.path.exists(path):
        USER_DEFINITION_LIST=json.loads(helper.readFile(path))
示例#11
0
    def test_saveToFileCorrect(self):
        ''' Tests if saveToFile changes the input it saves. '''
        file = 'testsavetofile'
        finput = 'Testing if this file gets modified.'
        save = helper.saveToFile(file, finput)
        actual = helper.readFile(file)
        expected_same = finput
        expected_diff = 'Different input.'

        self.assertEqual(actual, expected_same)
        self.assertNotEqual(actual, expected_diff)
示例#12
0
  def test_getContractsThreeCols(self):
    ''' Testing that three columns are correctly gathered to Contracts. '''
    execute = datagather.getContracts(self.cur, self.addresses)

    # To check is the data is added.
    self.cur.execute("""SELECT address, sizeofcode, transferred 
                        FROM testdatabase.contracts""")
    self.cur.connection.commit()
    actual   = str(self.cur.fetchall())
    expected = helper.readFile('getContractsOutput')

    self.assertEqual(actual, expected)
 def input_filename_done(self, path, name):
     filePath = os.path.join(path, name)
     print(filePath)
     if os.path.exists(filePath):
         sublime.error_message("Unable to create file, file exists.")
     else:
         # save
         file_name = os.path.basename(os.path.splitext(name)[0])
         template_content = helper.readFile(self.selected_template)
         template_content = template_content.replace("Template", file_name[0].upper() + file_name[1:])
         template_content = template_content.replace("template", file_name[0].lower() + file_name[1:])
         helper.writeFile(filePath, template_content)
         v = sublime.active_window().open_file(filePath)
         # done
         sublime.status_message("js file create done!")
         sublime.message_dialog("js file create done!")
示例#14
0
 def on_done(self, path, name):
     filePath = os.path.join(path, name)
     if os.path.exists(filePath):
         sublime.error_message("Unable to create file, file exists.")
     else:
         # load template file
         tmplPath = os.path.join(CUR_PATH,LIB_PATH, "lua.tmpl")
         code = helper.readFile(tmplPath)
         # add attribute
         settings = helper.loadSettings("quickx")
         format = settings.get("date_format", "%Y-%m-%d %H:%M:%S")
         date = datetime.datetime.now().strftime(format)
         code = code.replace("${date}", date)
         attr = settings.get("template_attr", {})
         for key in attr:
             code = code.replace("${%s}" % (key), attr.get(key, ""))
         # save
         helper.writeFile(filePath, code)
         v=sublime.active_window().open_file(filePath)
         # cursor
         v.run_command("insert_snippet",{"contents":code})
         sublime.status_message("Lua file create success!")
示例#15
0
def start(action):
    # 读取今天已经抓取过的url
    global fetched_url_list
    json_txt = helper.readFile('./logs/goat-%s.json' % helper.today())
    try:
        if json_txt:
            fetched_url_list = json.loads(json_txt)
    except:
        fetched_url_list = []
    f = open('./keyword.json')
    txt = f.read()
    f.close()
    key_list = json.loads(txt)
    # 去重
    # key_list = list(set(key_list))
    key_list = helper.delRepeat(key_list)
    crawl_counter = mongo.get_crawl_counter(platform)
    # 创建一个队列用来保存进程获取到的数据
    q = Queue()
    # 有错误的页面链接
    error_page_url_queue = Queue()

    # TODO:
    key_list = ['DUNK']
    for key in key_list:
        key = key.replace('\n', '')
        helper.log('[INFO] now key = ' + key, platform)
        # 先取男鞋 价格从低到高
        if fetch_page(1, 'PRICE_LOW_HIGH', key, q, error_page_url_queue,
                      crawl_counter):
            helper.log('[INFO] => fetch_page is done, 1, PRICE_LOW_HIGH',
                       platform)
        # 先取男鞋 价格从高到低
        if fetch_page(1, 'PRICE_HIGH_LOW', key, q, error_page_url_queue,
                      crawl_counter):
            helper.log('[INFO] => fetch_page is done, 1, PRICE_HIGH_LOW',
                       platform)
        # 先取女鞋 价格从低到高
        if fetch_page(2, 'PRICE_LOW_HIGH', key, q, error_page_url_queue,
                      crawl_counter):
            helper.log('[INFO] => fetch_page is done, 2, PRICE_LOW_HIGH',
                       platform)
        # 先取女鞋 价格从高到低
        if fetch_page(2, 'PRICE_HIGH_LOW', key, q, error_page_url_queue,
                      crawl_counter):
            helper.log('[INFO] => fetch_page is done, 2, PRICE_HIGH_LOW',
                       platform)
        # 先取青少年鞋 价格从低到高
        if fetch_page(5, 'PRICE_LOW_HIGH', key, q, error_page_url_queue,
                      crawl_counter):
            helper.log('[INFO] => fetch_page is done, 5, PRICE_LOW_HIGH',
                       platform)
        # 先取青少年鞋 价格从高到低
        if fetch_page(5, 'PRICE_HIGH_LOW', key, q, error_page_url_queue,
                      crawl_counter):
            helper.log('[INFO] => fetch_page is done, 5, PRICE_HIGH_LOW',
                       platform)
        #     # 先取婴儿鞋 价格从低到高
        #     if fetch_page(6, 'PRICE_LOW_HIGH', key, q, error_page_url_queue, crawl_counter):
        #         helper.log('[INFO] => fetch_page is done, 6, PRICE_LOW_HIGH', platform)
        #         # 先取婴儿鞋 价格从高到低
        #         fetch_page(6, 'PRICE_HIGH_LOW', key, q, error_page_url_queue, crawl_counter)
        #         helper.log('[INFO] => fetch_page is done, 6, PRICE_HIGH_LOW', platform)

    # goods_spider = GoodsSpider('https://www.goat.com/sneakers/force-savage-pro-baseball-cleat-880144-410', 1, Queue(), crawl_counter)
    # goods_spider.start()

    # 处理出错的链接
    # while not error_page_url_queue.empty():
    #     error_page_url_list = []
    #     while not error_page_url_queue.empty():
    #         error_page_url_list.append(error_page_url_queue.get())

    #     error_page_men_url_list = [{'url': url_data.get('url'), 'count': url_data.get('count')} for url_data in error_page_url_list if url_data.get('gender') == 1]
    #     fetch_page([{'url': url_data.get('url'), 'count': url_data.get('count')} for url_data in error_page_men_url_list], 1, q, error_page_url_queue, {
    #         'mnid': 'men_shoes',
    #         'Ns': 'sku.bestSeller | 1',
    #         'isAjax': 'true'
    #     }, crawl_counter)
    helper.log('done', platform)
示例#16
0
    if args.algorithm not in algorithms:
        raise AssertionError(
            f'Incorrect algorithm specified - "{args.algorithm}". Try using --help.'
        )
    return args


if __name__ == '__main__':
    algorithms = [
        'epsilon-greedy', 'ucb', 'kl-ucb', 'thompson-sampling',
        'thompson-sampling-with-hint'
    ]
    args = parseArguements(algorithms)

    # Get the true means from the file
    means_true = helper.readFile(args.instance)

    # Call to the appropriate function
    regret = None
    if args.algorithm == 'epsilon-greedy':
        regret = epsilonGreedy(args.randomSeed, args.horizon, means_true,
                               args.epsilon, args.verbose)
    elif args.algorithm == 'ucb':
        regret = ucb(args.randomSeed, args.horizon, means_true, args.verbose)
    elif args.algorithm == 'kl-ucb':
        regret = ucbKL(args.randomSeed, args.horizon, means_true, args.verbose)
    elif args.algorithm == 'thompson-sampling':
        regret = thompsonSampling(args.randomSeed, args.horizon, means_true,
                                  args.verbose)
    elif args.algorithm == 'thompson-sampling-with-hint':
        regret = thompsonSamplingWithHint(args.randomSeed, args.horizon,
示例#17
0
profs = json.loads(open('profiles.json','r').read())
IDs = []
for prof in profs:
	IDs.append(prof['id'])

cIDs = json.loads(open('ML_public_comps_no_num.py', 'r').read())
cIDs = cIDs[:] # Mar 20, 5pm with mine
cIDs = cIDs[13:]	# Mar 20, 5pm with Eyal
cIDs = cIDs[29:]	# Mar 20, 11pm with mine
for cID in cIDs:
	cID['numIDs'] = str(h.countCIDs(api, cID, IDs))
	with open('ML_public_comps.py', 'a') as f:
		f.write(json.dumps(cID) + ',\n')
# saved as ML_public_comps.py

pIDs = h.readFile('./IDs_all.txt')
#pIDs2 = pIDs[0:845]	# Mar 4, 7pm with mine. Saved as profiles_0_845, size 725
#pIDs2 = pIDs[845:]		# Mar 5, 2pm with Eyal, saved as profiles_0_1083, size 927
#pIDs2 = pIDs[1083:]	# Mar 5, 3pm with Elisha, saved as profiles_0_1645, size 1458
#pIDs2 = pIDs[1645:]	# Mar 5, 4pm with Karthik, saved as profiles_0_1869, size 1639

#pIDs2 = pIDs[1870:]	# Mar 6, 3pm with Elisha, saved as profiles_0_2620, size 2391
#pIDs2 = pIDs[2621:]	# Mar 6, 4pm with Dawei, saved as profiles_0_3624, size 3395
#pIDs2 = pIDs[3625:]	# Mar 6, 4pm with Dan, saved as profiles_0_4872, size 4643
#pIDs2 = pIDs[4873:]	# Mar 6, 5pm with Kevin, saved as profiles_0_4972, ID4973 is bad
#pIDs2 = pIDs[4974:]	# continue with Kevin, saved as profiles_0_5040, internal error
#pIDs2 = pIDs[5041:]	# continue with Kevin, saved as profiles_0_5130, internal error
#pIDs2 = pIDs[5131:]	# continue with Kevin, saved as profiles_0_5334, internal error
#pIDs2 = pIDs[5335:]	# continue with Kevin, saved as profiles_0_5362, internal error
#pIDs2 = pIDs[5363:]	# continue with Kevin, saved as profiles_0_5626, size 5423
#pIDs2 = pIDs[5627:]	# Mar 6, 6pm with Eyal, saved as profiles_0_5677, internal error
示例#18
0
    def test_readFileType(self):
        ''' Tests if readFile returns the correct type. '''
        actual = type(helper.readFile('test_readFile'))
        expected = type('string')

        self.assertEqual(actual, expected)
str7 = "7. Exit\n"
while (1):
    print
    try:
        choise = int(raw_input(str1 + str2 + str3 + str4 + str5 + str6 + str7))
    except ValueError:
        print "Not a number"

    if (choise == 1):
        print "Generating key"
        key = crypto1.randkeygen(16 * 8)
        print "key: ", key

        #filename=str(raw_input("please type filename to be encrypted: "))
        filename = "test"
        msg = helper.readFile(filename + ".txt")
        IV = crypto1.IVGen(16 * 8)
        cipher = crypto1.encrypt(msg, key, 16 * 8, "CBC", IV)
        helper.saveCipher(filename + "_AES128.txt", cipher)
        print "Cipher saved at file: " + filename + ".txt"

    elif (choise == 2):
        #filename=str(raw_input("please type filename to be decrypted: "))
        strcipher = helper.readFile(filename + "_AES128.txt")

        tmp_cipher = strcipher.split(",")
        cipher = [int(i) for i in tmp_cipher]
        plain = crypto1.decrypt(cipher, key, 16 * 8, "CBC", IV)
        helper.saveFile(filename + "_AES128_plain.txt", plain)
        print "Decrypted file saved as: " + filename + "_AES128_plain.txt"