def test_getBsObjectContent(self):
''' Testing if getBsObject returns the right content. '''
actual = (helper.getBsObject(self.url)).prettify()
expected_same = helper.readFile('pythonscraping_example.html')
expected_diff = helper.readFile('pythonscraping_wrong_example.html')
self.assertEqual(actual, expected_same)
self.assertNotEqual(actual, expected_diff)
def test_readFileCorrect(self):
''' Tests if the file is read accurately. '''
actual_simp = helper.readFile('testfile')
actual_utf8 = helper.readFile('testfile_strange')
expected_simp = 'Testing if this file gets modified.'
expected_utf8 = 'Testing with strange input √√«ß‹∫‹∑ ß∆ñ∆œ∑ß...'
self.assertEqual(actual_simp, expected_simp)
self.assertEqual(actual_utf8, expected_utf8)
def test_removeComments(self):
''' Tests if the outputfile has removed comments as it should have. '''
orginalfile = helper.readFile('removecommentsfile')
correctfile = helper.readFile('correct_removecomments')
outputfile = helper.removeComments(orginalfile)
actual = hashlib.md5(str(outputfile).encode()).hexdigest()
expected = hashlib.md5(str(correctfile).encode()).hexdigest()
self.assertEqual(actual, expected)
def create_jsc_folder(self):
jsc_folder = self.target_directory + '/' + 'jsc'
helper.create_directory(jsc_folder)
helper.create_directory(jsc_folder + '/cocos2d_js')
helper.copytree(sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/jsc/cocos2d_js", jsc_folder + '/cocos2d_js')
# create complier_config.json
complier_config_file = sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/jsc/compiler_config.json"
complier_config_content = helper.readFile(complier_config_file)
helper.writeFile(jsc_folder + "/complier_config.json", complier_config_content)
# create generate_jsc.py
generate_jsc_file = sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/jsc/generate_jsc.py"
generate_jsc_content = helper.readFile(generate_jsc_file)
generate_jsc_content = generate_jsc_content.replace("%BINDING_JS_FOLDER%", sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/jsc/bindings")
generate_jsc_content = generate_jsc_content.replace("%BIN_EXE_PATH%", sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/jsc/bin/jsbcc")
helper.writeFile(jsc_folder + "/generate_jsc.py", generate_jsc_content)
def dealWithFile(filepath): #get filename to mkdir base_name = os.path.basename(filepath) base_name = os.path.splitext(base_name)[0] parent_path = os.path.abspath(os.path.join(output_path,base_name)) if(not os.path.exists(parent_path)): os.mkdir(parent_path) readfp = helper.readFile(filepath).splitlines() while len(readfp)>0: line_text = readfp.pop(0); #first match className result = re.match(r'.*@module (\w+)', line_text) if result: className = result.group(1) dealWithModule(className, readfp, parent_path) else: #match function result = re.match(r'.*@function .* (\w+)', line_text) if result: funcName = result.group(1) #too many function named create,deal it single if funcName == "create": dealWithCreate(className, readfp, parent_path) elif funcName != className: #ignore constructor dealWithFunction(funcName, readfp, parent_path, className)
def create_ant_folder(self):
ant_folder = self.target_directory + '/' + 'ant'
helper.create_directory(ant_folder)
# create build.xml
build_file = sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/ant/build.xml"
build_content = helper.readFile(build_file)
build_content = build_content.replace("%MODE%", helper.loadSettings("Cocos2dJSDev").get("ant").get("mode"))
build_content = build_content.replace("%DEBUG%", helper.loadSettings("Cocos2dJSDev").get("ant").get("debug"))
build_content = build_content.replace("%OUTPUT.js%", helper.loadSettings("Cocos2dJSDev").get("ant").get("output"))
build_content = build_content.replace("%COMPLIER_JAR_PATH%", sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/ant")
build_content = build_content.replace("%COCOS2D_ROOT_PATH%", helper.loadSettings("Cocos2dJSDev").get("cocos2d_html5_root"))
helper.writeFile(ant_folder + "/build.xml", build_content)
# create cocos2d_extern.js
cocos2d_externs_file = sublime.packages_path() + "/Cocos2dJSDev/cocos2d_js_lib/ant/cocos2d_externs.js"
cocos2d_externs_content = helper.readFile(cocos2d_externs_file)
helper.writeFile(ant_folder + "/cocos2d_externs.js", cocos2d_externs_content)
def init():
global TEMP_PATH
TEMP_PATH=sublime.packages_path()+"/User/QuickXDev.cache"
global USER_DEFINITION_LIST
path=os.path.join(TEMP_PATH,"user_definition.json")
if os.path.exists(path):
USER_DEFINITION_LIST=json.loads(helper.readFile(path))
def init():
global TEMP_PATH
TEMP_PATH=sublime.packages_path()+"/User/quick-comminuty-dev.cache"
global DEFINITION_LIST
DEFINITION_LIST=json.loads(definition.data)
global USER_DEFINITION_LIST
path=os.path.join(TEMP_PATH,"user_definition.json")
if os.path.exists(path):
USER_DEFINITION_LIST=json.loads(helper.readFile(path))
def run(self, files):
self.target_file = files[0]
resource_content = helper.readFile(self.target_file)
resources = re.findall(r"\s*{src:\s*(.*)}", resource_content)
# print param_list
del RESOURCE_LIST[:]
for resource in resources:
RESOURCE_LIST.append(resource)
sublime.message_dialog("Update Resource List Done!")
def init():
global TEMP_PATH,DEFINITION_LIST,USER_DEFINITION_LIST
TEMP_PATH=sublime.packages_path()+"/User/lufylegendDev.cache"
lufylegend_root = checkRoot(False)
if lufylegend_root:
initDefinition(lufylegend_root)
path=os.path.join(TEMP_PATH,"user_definition.json")
if os.path.exists(path):
USER_DEFINITION_LIST=json.loads(helper.readFile(path))
def test_saveToFileCorrect(self):
''' Tests if saveToFile changes the input it saves. '''
file = 'testsavetofile'
finput = 'Testing if this file gets modified.'
save = helper.saveToFile(file, finput)
actual = helper.readFile(file)
expected_same = finput
expected_diff = 'Different input.'
self.assertEqual(actual, expected_same)
self.assertNotEqual(actual, expected_diff)
def test_getContractsThreeCols(self):
''' Testing that three columns are correctly gathered to Contracts. '''
execute = datagather.getContracts(self.cur, self.addresses)
# To check is the data is added.
self.cur.execute("""SELECT address, sizeofcode, transferred
FROM testdatabase.contracts""")
self.cur.connection.commit()
actual = str(self.cur.fetchall())
expected = helper.readFile('getContractsOutput')
self.assertEqual(actual, expected)
def input_filename_done(self, path, name):
filePath = os.path.join(path, name)
print(filePath)
if os.path.exists(filePath):
sublime.error_message("Unable to create file, file exists.")
else:
# save
file_name = os.path.basename(os.path.splitext(name)[0])
template_content = helper.readFile(self.selected_template)
template_content = template_content.replace("Template", file_name[0].upper() + file_name[1:])
template_content = template_content.replace("template", file_name[0].lower() + file_name[1:])
helper.writeFile(filePath, template_content)
v = sublime.active_window().open_file(filePath)
# done
sublime.status_message("js file create done!")
sublime.message_dialog("js file create done!")
def on_done(self, path, name):
filePath = os.path.join(path, name)
if os.path.exists(filePath):
sublime.error_message("Unable to create file, file exists.")
else:
# load template file
tmplPath = os.path.join(CUR_PATH,LIB_PATH, "lua.tmpl")
code = helper.readFile(tmplPath)
# add attribute
settings = helper.loadSettings("quickx")
format = settings.get("date_format", "%Y-%m-%d %H:%M:%S")
date = datetime.datetime.now().strftime(format)
code = code.replace("${date}", date)
attr = settings.get("template_attr", {})
for key in attr:
code = code.replace("${%s}" % (key), attr.get(key, ""))
# save
helper.writeFile(filePath, code)
v=sublime.active_window().open_file(filePath)
# cursor
v.run_command("insert_snippet",{"contents":code})
sublime.status_message("Lua file create success!")
def start(action):
# 读取今天已经抓取过的url
global fetched_url_list
json_txt = helper.readFile('./logs/goat-%s.json' % helper.today())
try:
if json_txt:
fetched_url_list = json.loads(json_txt)
except:
fetched_url_list = []
f = open('./keyword.json')
txt = f.read()
f.close()
key_list = json.loads(txt)
# 去重
# key_list = list(set(key_list))
key_list = helper.delRepeat(key_list)
crawl_counter = mongo.get_crawl_counter(platform)
# 创建一个队列用来保存进程获取到的数据
q = Queue()
# 有错误的页面链接
error_page_url_queue = Queue()
# TODO:
key_list = ['DUNK']
for key in key_list:
key = key.replace('\n', '')
helper.log('[INFO] now key = ' + key, platform)
# 先取男鞋 价格从低到高
if fetch_page(1, 'PRICE_LOW_HIGH', key, q, error_page_url_queue,
crawl_counter):
helper.log('[INFO] => fetch_page is done, 1, PRICE_LOW_HIGH',
platform)
# 先取男鞋 价格从高到低
if fetch_page(1, 'PRICE_HIGH_LOW', key, q, error_page_url_queue,
crawl_counter):
helper.log('[INFO] => fetch_page is done, 1, PRICE_HIGH_LOW',
platform)
# 先取女鞋 价格从低到高
if fetch_page(2, 'PRICE_LOW_HIGH', key, q, error_page_url_queue,
crawl_counter):
helper.log('[INFO] => fetch_page is done, 2, PRICE_LOW_HIGH',
platform)
# 先取女鞋 价格从高到低
if fetch_page(2, 'PRICE_HIGH_LOW', key, q, error_page_url_queue,
crawl_counter):
helper.log('[INFO] => fetch_page is done, 2, PRICE_HIGH_LOW',
platform)
# 先取青少年鞋 价格从低到高
if fetch_page(5, 'PRICE_LOW_HIGH', key, q, error_page_url_queue,
crawl_counter):
helper.log('[INFO] => fetch_page is done, 5, PRICE_LOW_HIGH',
platform)
# 先取青少年鞋 价格从高到低
if fetch_page(5, 'PRICE_HIGH_LOW', key, q, error_page_url_queue,
crawl_counter):
helper.log('[INFO] => fetch_page is done, 5, PRICE_HIGH_LOW',
platform)
# # 先取婴儿鞋 价格从低到高
# if fetch_page(6, 'PRICE_LOW_HIGH', key, q, error_page_url_queue, crawl_counter):
# helper.log('[INFO] => fetch_page is done, 6, PRICE_LOW_HIGH', platform)
# # 先取婴儿鞋 价格从高到低
# fetch_page(6, 'PRICE_HIGH_LOW', key, q, error_page_url_queue, crawl_counter)
# helper.log('[INFO] => fetch_page is done, 6, PRICE_HIGH_LOW', platform)
# goods_spider = GoodsSpider('https://www.goat.com/sneakers/force-savage-pro-baseball-cleat-880144-410', 1, Queue(), crawl_counter)
# goods_spider.start()
# 处理出错的链接
# while not error_page_url_queue.empty():
# error_page_url_list = []
# while not error_page_url_queue.empty():
# error_page_url_list.append(error_page_url_queue.get())
# error_page_men_url_list = [{'url': url_data.get('url'), 'count': url_data.get('count')} for url_data in error_page_url_list if url_data.get('gender') == 1]
# fetch_page([{'url': url_data.get('url'), 'count': url_data.get('count')} for url_data in error_page_men_url_list], 1, q, error_page_url_queue, {
# 'mnid': 'men_shoes',
# 'Ns': 'sku.bestSeller | 1',
# 'isAjax': 'true'
# }, crawl_counter)
helper.log('done', platform)
if args.algorithm not in algorithms:
raise AssertionError(
f'Incorrect algorithm specified - "{args.algorithm}". Try using --help.'
)
return args
if __name__ == '__main__':
algorithms = [
'epsilon-greedy', 'ucb', 'kl-ucb', 'thompson-sampling',
'thompson-sampling-with-hint'
]
args = parseArguements(algorithms)
# Get the true means from the file
means_true = helper.readFile(args.instance)
# Call to the appropriate function
regret = None
if args.algorithm == 'epsilon-greedy':
regret = epsilonGreedy(args.randomSeed, args.horizon, means_true,
args.epsilon, args.verbose)
elif args.algorithm == 'ucb':
regret = ucb(args.randomSeed, args.horizon, means_true, args.verbose)
elif args.algorithm == 'kl-ucb':
regret = ucbKL(args.randomSeed, args.horizon, means_true, args.verbose)
elif args.algorithm == 'thompson-sampling':
regret = thompsonSampling(args.randomSeed, args.horizon, means_true,
args.verbose)
elif args.algorithm == 'thompson-sampling-with-hint':
regret = thompsonSamplingWithHint(args.randomSeed, args.horizon,
profs = json.loads(open('profiles.json','r').read())
IDs = []
for prof in profs:
IDs.append(prof['id'])
cIDs = json.loads(open('ML_public_comps_no_num.py', 'r').read())
cIDs = cIDs[:] # Mar 20, 5pm with mine
cIDs = cIDs[13:] # Mar 20, 5pm with Eyal
cIDs = cIDs[29:] # Mar 20, 11pm with mine
for cID in cIDs:
cID['numIDs'] = str(h.countCIDs(api, cID, IDs))
with open('ML_public_comps.py', 'a') as f:
f.write(json.dumps(cID) + ',\n')
# saved as ML_public_comps.py
pIDs = h.readFile('./IDs_all.txt')
#pIDs2 = pIDs[0:845] # Mar 4, 7pm with mine. Saved as profiles_0_845, size 725
#pIDs2 = pIDs[845:] # Mar 5, 2pm with Eyal, saved as profiles_0_1083, size 927
#pIDs2 = pIDs[1083:] # Mar 5, 3pm with Elisha, saved as profiles_0_1645, size 1458
#pIDs2 = pIDs[1645:] # Mar 5, 4pm with Karthik, saved as profiles_0_1869, size 1639
#pIDs2 = pIDs[1870:] # Mar 6, 3pm with Elisha, saved as profiles_0_2620, size 2391
#pIDs2 = pIDs[2621:] # Mar 6, 4pm with Dawei, saved as profiles_0_3624, size 3395
#pIDs2 = pIDs[3625:] # Mar 6, 4pm with Dan, saved as profiles_0_4872, size 4643
#pIDs2 = pIDs[4873:] # Mar 6, 5pm with Kevin, saved as profiles_0_4972, ID4973 is bad
#pIDs2 = pIDs[4974:] # continue with Kevin, saved as profiles_0_5040, internal error
#pIDs2 = pIDs[5041:] # continue with Kevin, saved as profiles_0_5130, internal error
#pIDs2 = pIDs[5131:] # continue with Kevin, saved as profiles_0_5334, internal error
#pIDs2 = pIDs[5335:] # continue with Kevin, saved as profiles_0_5362, internal error
#pIDs2 = pIDs[5363:] # continue with Kevin, saved as profiles_0_5626, size 5423
#pIDs2 = pIDs[5627:] # Mar 6, 6pm with Eyal, saved as profiles_0_5677, internal error
def test_readFileType(self):
''' Tests if readFile returns the correct type. '''
actual = type(helper.readFile('test_readFile'))
expected = type('string')
self.assertEqual(actual, expected)
str7 = "7. Exit\n"
while (1):
print
try:
choise = int(raw_input(str1 + str2 + str3 + str4 + str5 + str6 + str7))
except ValueError:
print "Not a number"
if (choise == 1):
print "Generating key"
key = crypto1.randkeygen(16 * 8)
print "key: ", key
#filename=str(raw_input("please type filename to be encrypted: "))
filename = "test"
msg = helper.readFile(filename + ".txt")
IV = crypto1.IVGen(16 * 8)
cipher = crypto1.encrypt(msg, key, 16 * 8, "CBC", IV)
helper.saveCipher(filename + "_AES128.txt", cipher)
print "Cipher saved at file: " + filename + ".txt"
elif (choise == 2):
#filename=str(raw_input("please type filename to be decrypted: "))
strcipher = helper.readFile(filename + "_AES128.txt")
tmp_cipher = strcipher.split(",")
cipher = [int(i) for i in tmp_cipher]
plain = crypto1.decrypt(cipher, key, 16 * 8, "CBC", IV)
helper.saveFile(filename + "_AES128_plain.txt", plain)
print "Decrypted file saved as: " + filename + "_AES128_plain.txt"
