def minimize(filename): trolleybuses = utils.readfile(filename) minimum_prec, minimum = 15000000000, 10000000000 max_interval = 20 while (minimum_prec > minimum): T_prime, alpha = {}, {} trolleybus = [] for key in trolleybuses.keys(): T_prime[key] = utils.add_one(trolleybuses, trolleybuses[key]) for key in T_prime.keys(): T_prime[key].sort() alpha[key] = utils.calc_sum(T_prime[key]) print(alpha) minimum_key = min(alpha, key=alpha.get) minimum_prec = minimum minimum = alpha[minimum_key] print(minimum_key, minimum) trolleybuses[minimum_key] = [x + 1 for x in trolleybuses[minimum_key]] visualise(trolleybuses) plt.title("New time-table") visualise(utils.readfile("sources/stefan_cel_mare.txt")) plt.title("Old time-table") plt.show() return trolleybuses
def main(timed_run_params:TimedRunParams, tool_run:ToolRunParams) -> (RunStats, int): # TODO: add memory limit logging.info('timed_run.main') stats_file_name = get_tmp_file_name() exec_log_file = get_tmp_file_name() rc, out, err = \ execute_shell('{runsolver} -o {tool_log} -v {stats_file} -w {exec_log} -W {time_limit} ' '{tool_cmd}' .format(runsolver=RUN_SOLVER_EXEC, tool_log=tool_run.log_file, stats_file=stats_file_name, exec_log=exec_log_file, time_limit=str(timed_run_params.time_limit_sec), tool_cmd=tool_run.to_cmd_str())) logging.info(readfile(exec_log_file)) # TODO: this should also be logged in the DB assert rc == 0, 'timed run failed: rc={rc}, \nout={out}, \nerr={err}'\ .format(rc=str(rc), out=out, err=err) tool_rc = get_tool_rc(readfile(exec_log_file)) stats = parse_stats(readfile(stats_file_name)) os.remove(stats_file_name) os.remove(exec_log_file) return stats, tool_rc
def compileHTML(filepath, needCompress): """编译html文件 Arguments: - `filepath`: """ if not os.path.exists(filepath): return False tpl = utils.readfile(filepath) log.log("Compile for " + filepath + ".") LINK_TOKEN = "<link.* href=['\"](?!http|https)(.*?\.css)['\"]" # 不包含以http/https开头的资源 SCRIPT_TOKEN = "<script.* src=['\"](?!http|https)(.*?\.js)['\"]" iters = re.finditer(LINK_TOKEN, tpl) for i in reversed(list(iters)): path = i.group(1) path = compileCommon(path, "local", True) # 内部可能有替换的变量 if not conf.getConfig().get("disableAutoTimestamp"): tpl = tpl[0 : i.start(1)] + i.group(1) + "?t=" + getFileTimeStamp(path, filepath) + tpl[i.end(1) :] iters = re.finditer(SCRIPT_TOKEN, tpl) for i in reversed(list(iters)): path = i.group(1) path = compileCommon(path, "local", True) # 内部可能有替换的变量 if not conf.getConfig().get("disableAutoTimestamp"): tpl = tpl[0 : i.start(1)] + i.group(1) + "?t=" + getFileTimeStamp(path, filepath) + tpl[i.end(1) :] return tpl
def join(self): arg = pydev.AutoArg() test_num = int(arg.option('testnum', -1)) input_filename = arg.option('f') movie_dir = arg.option('m') slot_output_filename = arg.option('s') output_filename = arg.option('o') coder_output_filename = arg.option('c') data = utils.readfile(file(input_filename), test_num=test_num) extractor = MovieLensRankingFeatureExtractor(movie_dir) writer = sf.SlotFileWriter(output_filename) for user_id, item_id, click in tqdm.tqdm(data): writer.begin_instance(click) extractor.begin(user_id, item_id) ps = extractor.processes() for p in ps: slot, lst = p() writer.write_slot(slot, lst) writer.end_instance() extractor.save(coder_output_filename, slot_output_filename) writer.summary()
def getData(token): """ Arguments: - `token`: """ data = getRawData(token) if len(data): data = json.loads(data) else: data = {} dorepeat(data); commonpath = BASE + COMMON_TOKEN + SUFFIX if os.path.exists(commonpath): commondata = json.loads( utils.readfile( commonpath ) ) else: commondata = {} data.update(commondata) multoken = token.split('/') data.update({'_token':token.replace('/','_')}) data.update({'_folder':multoken[0]}) data.update({'_subtoken':multoken[1] if len(multoken)>1 else ""}) return data
def _css(self): ''' handle css r.js会对不同目录下CSS合并时的URL进行修正,因而对于@something@开头的路径会被认为是相对路径, 产生修正错误,解决方案是先对所有CSS文件进行变量替换,时间戳添加,再由r.js合并。这会降低处理速 度但可以解决该问题。 考虑到速度,此过程仅支持在build时进行,开发服务器访问时不能使用。 所有静态资源路径都应该使用绝对路径,避免在CSS中引用相对路径的图像资源。 ''' #搜索所有CSS文件 all_css_files = utils.FileSearcher(r'\.css$',self._build_css_dir,relative = False).search() #替换和加时间戳 for dst in all_css_files: try: content = utils.readfile(dst) content = all_url(content,os.path.dirname(dst)) content = replace(content,self._target) utils.writefile(dst,content) except Exception,e: if self._force: log.error('[css]%s'%e) else: raise e
def parseTpl(token, data={}, noGetTpl=False, isbuild=False): """ """ if not noGetTpl: tpl = token + ".tpl" else: tpl = token if not noGetTpl: if isbuild: body = build_jinjaenv.get_template(tpl) else: body = jinjaenv.get_template(tpl) elif os.path.exists(tpl): body = Template(utils.readfile(tpl)) else: return "" if not len(data): data = mgr.getData(token) try: body = body.render(data) except TemplateNotFound as e: return "Template %s not found" % (str(e),) except TemplateSyntaxError as e: # [email protected]:捕获语法错误,很有用 return "Template %s:%d Syntax Error:%s" % (e.filename, e.lineno, e.message) except Exception as e: print e return "" return body
def _tpl(self): ''' handle tempaltes 模板仅需加时间戳和变量替换。 这里需要加入额外的{compile_dir}文件夹下的文本文件。 ''' fs = utils.FileSearcher(r'\.%s$'%C('template_ext'),self._build_tpl_dir,relative = False) tpls = fs.search() if self._compile_dir: nfs = utils.FileSearcher(r'.+',self._build_compile_dir,relative = False) compile_files = nfs.search() for f in compile_files: if not utils.isBinary(f): tpls.insert(0,f) for tpl in tpls: try: content = utils.readfile(tpl) #模板的静态资源相对目录应该写死为cwd,即资源路径应该始终是绝对路径 content = allt(content,self._build_dir,force_abspath = False) content = replace(content,self._target) content = removeCssDepsDeclaration(content) utils.writefile(tpl,content) except Exception,e: if self._force: log.error('[tpl]%s'%e) else: raise e
def step_t_5(a_base, B, p1): if p1 <= int(root(B, 5)): ### Посчет времени работы start_time = time.time() ### n_list = [] l = 5 if p1 % 4 == 3: n_list = [p1] q_3k4 = readfile("primes/4k+3.txt") sign_p1 = [] for a in a_base: sign_p1.append(numth.jacobi(a, p1)) for q in q_3k4: if q % 24 == p1 % 24: sign_q = [] for a in a_base: sign_q.append(numth.jacobi(a, q)) if sign_q == sign_p1: n_list.append(q) # else: # for p in primes: ### total_time = "--- %s seconds ---\n" % (time.time() - start_time) ### return np.array(n_list) else: print(f"Value Error: p1 > {int(root(B, 5))}")
def compileHTML(filepath, needCompress): """编译html文件 Arguments: - `filepath`: """ if not os.path.exists(filepath): return False tpl = utils.readfile(filepath) log.log('Compile for ' + filepath + '.') LINK_TOKEN = '<link.* href=[\'"](?!http|https)(.*?\.css)[\'"]' # 不包含以http/https开头的资源 SCRIPT_TOKEN = '<script.* src=[\'"](?!http|https)(.*?\.js)[\'"]' iters = re.finditer(LINK_TOKEN, tpl) for i in reversed(list(iters)): path = i.group(1) path = compileCommon(path, 'local', True) #内部可能有替换的变量 if not conf.getConfig().get('disableAutoTimestamp'): tpl = tpl[0:i.start(1)] + i.group(1) + '?t=' + getFileTimeStamp( path, filepath) + tpl[i.end(1):] iters = re.finditer(SCRIPT_TOKEN, tpl) for i in reversed(list(iters)): path = i.group(1) path = compileCommon(path, 'local', True) #内部可能有替换的变量 if not conf.getConfig().get('disableAutoTimestamp'): tpl = tpl[0:i.start(1)] + i.group(1) + '?t=' + getFileTimeStamp( path, filepath) + tpl[i.end(1):] return tpl
def parseTpl(token, data={}, noGetTpl=False, isbuild=False): """ """ if not noGetTpl: tpl = token + '.tpl' else: tpl = token if not noGetTpl: if isbuild: body = build_jinjaenv.get_template(tpl) else: body = jinjaenv.get_template(tpl) elif os.path.exists(tpl): body = Template(utils.readfile(tpl)) else: return '' if not len(data): data = mgr.getData(token) try: body = body.render(data) except TemplateNotFound as e: return 'Template %s not found' % (str(e), ) except TemplateSyntaxError as e: #[email protected]:捕获语法错误,很有用 return 'Template %s:%d Syntax Error:%s' % (e.filename, e.lineno, e.message) except Exception as e: print e return '' return body
def compileCommon(filepath, token, force=False): """通用编译方法 编译 @tm:file_path@为6位时间戳 Arguments: - `content`: """ if force: content = filepath else: if not os.path.exists(filepath): return False ftype = filepath.split('.')[-1] if not ftype in ['html', 'htm', 'css', 'js', 'tpl', 'jsp']: return False content = utils.readfile(filepath) TM_TOKEN = '@tm:(.*?)@' DATE_TOKEN = '@date@' COMMON_TOKEN = '@(.*?)@' iters = re.finditer(TM_TOKEN, content) for i in reversed(list(iters)): content = content[0:i.start(0)] + getFileTimeStamp( i.group(1), filepath) + content[i.end(0):] iters = re.finditer(DATE_TOKEN, content) for i in reversed(list(iters)): content = content[0:i.start(0)] + utils.getDate() + content[i.end(0):] iters = re.finditer(COMMON_TOKEN, content) for i in reversed(list(iters)): config = conf.getConfig() name = i.group(1) value = (token and config[token].get(name)) or config.get(name) if value: if value.find('{num}') != -1: num = (token and config[token].get('num')) or config.get('num') or '10' num = range(num + 1) substr100 = content[i.end(0):i.end(0) + 100] istimestamp = substr100.find('t=') if istimestamp != -1: #has timestamp try: tm = int(substr100[istimestamp + 2:istimestamp + 3]) except ValueError: continue if tm >= len(num): tm = tm - len(num) value = value.replace('{num}', str(tm)) else: global range_item value = value.replace('{num}', str(num[range_item])) range_item = range_item + 1 if range_item >= len(num): range_item = 0 content = content[0:i.start(0)] + value + content[i.end(0):] return content
def show_iteration_content(request, name, iteration): iteration_path = os.path.join(config.CATEGORY_BASEDIR, name, iteration) if os.path.isfile(iteration_path): instances, patterns = utils.readfile(iteration_path) context = { 'instances': instances, 'patterns': patterns, 'can_update': can_update(name, iteration) } return render(request, 'category/iteration_content.html', context) else: return HttpResponseNotFound('iteration %s/%s not exist!' % (name, iteration))
def RedisScript(spirit_name): #position = utils.seek2(log[lenseinfo_name]) process = os.popen("redis-server /etc/redis.conf > ./tempfile 2>&1") #process0 = os.popen("redis-server /etc/redis.conf") #print process.read() time.sleep(2) process1 = os.popen("redis-cli shutdown") return utils.readfile("./tempfile")
def save_build_env(): jailname = readfile(e('${OBJDIR}/jailname')) sh("mkdir -p ${SAVED_BUILD_ENV_DESTDIR}") sh("mkdir -p ${SAVED_BUILD_ENV_DESTDIR}/wrkdirs") sh("cp ${BE_ROOT}/repo-manifest ${SAVED_BUILD_ENV_DESTDIR}/") sh("cp -R ${WORLD_DESTDIR} ${SAVED_BUILD_ENV_DESTDIR}/") sh("cp -R ${DEBUG_ROOT} ${SAVED_BUILD_ENV_DESTDIR}/") sh("cp -R ${OBJDIR}/ports/wrkdirs/${jailname}-p/p/*.tbz ${SAVED_BUILD_ENV_DESTDIR}/wrkdirs/", nofail=True)
def dynamic(path): try: filename = utils.get_dynamic_path(path) return flask.render_template( 'base.jade', post=utils.readfile(filename)) except ValueError: print('no files with path {}'.format(path)) return flask.abort(404)
def compileCommon(filepath, token, force=False): """通用编译方法 编译 @tm:file_path@为6位时间戳 Arguments: - `content`: """ if force: content = filepath else: if not os.path.exists(filepath): return False ftype = filepath.split(".")[-1] if not ftype in ["html", "htm", "css", "js", "tpl", "jsp"]: return False content = utils.readfile(filepath) TM_TOKEN = "@tm:(.*?)@" DATE_TOKEN = "@date@" COMMON_TOKEN = "@(.*?)@" iters = re.finditer(TM_TOKEN, content) for i in reversed(list(iters)): content = content[0 : i.start(0)] + getFileTimeStamp(i.group(1), filepath) + content[i.end(0) :] iters = re.finditer(DATE_TOKEN, content) for i in reversed(list(iters)): content = content[0 : i.start(0)] + utils.getDate() + content[i.end(0) :] iters = re.finditer(COMMON_TOKEN, content) for i in reversed(list(iters)): config = conf.getConfig() name = i.group(1) value = (token and config[token].get(name)) or config.get(name) if value: if value.find("{num}") != -1: num = (token and config[token].get("num")) or config.get("num") or "10" num = range(num + 1) substr100 = content[i.end(0) : i.end(0) + 100] istimestamp = substr100.find("t=") if istimestamp != -1: # has timestamp try: tm = int(substr100[istimestamp + 2 : istimestamp + 3]) except ValueError: continue if tm >= len(num): tm = tm - len(num) value = value.replace("{num}", str(tm)) else: global range_item value = value.replace("{num}", str(num[range_item])) range_item = range_item + 1 if range_item >= len(num): range_item = 0 content = content[0 : i.start(0)] + value + content[i.end(0) :] return content
def load_extract_data(data_path, w2v_path, w2v_dim, sequence_length): def exEntities(list_s): start_positions = [] for i, token in enumerate(list_s): if token.find("/Entity") != -1: start_positions.append(i) return start_positions print("[LOADING] Extract data: ", data_path) data = utils.readfile(data_path) dict_origin_data = {} dict_tagged_data = {} tagged_sents = [] for line in data: sent = line[0] lbox_ids = ":".join(line[1:]) tagged_s = utils.pos_tag_no_e1e2(sent) tagged_sents.append(tagged_s) dict_origin_data[lbox_ids] = sent dict_tagged_data[lbox_ids] = tagged_s bow, embed = const_bow(tagged_sents, w2v_path, w2v_dim) print("[COMPLETE] Constructing BOW") o_sents = [] o_pos1 = [] o_pos2 = [] o_e1 = [] o_e2 = [] o_ids = [] for key_, tagged_s in dict_tagged_data.items(): if len(tagged_s) > sequence_length: continue entities = exEntities(tagged_s) for e1_pos, e2_pos in itertools.permutations(entities, 2): e1 = tagged_s[e1_pos].split("/En")[0] e2 = tagged_s[e2_pos].split("/En")[0] tmp_s = [] tmp_pos1 = [] tmp_pos2 = [] for idx, token in enumerate(tagged_s): if token in bow: tmp_s.append(bow[token]) tmp_pos1.append(pos_embed(e1_pos - idx)) tmp_pos2.append(pos_embed(e2_pos - idx)) sent_len = len(tagged_s) while len(tmp_s) != sequence_length: tmp_s.append(bow["<zero>"]) tmp_pos1.append(122) tmp_pos2.append(122) sent_len += 1 o_sents.append(tmp_s) o_pos1.append(tmp_pos1) o_pos2.append(tmp_pos2) o_e1.append(e1) o_e2.append(e2) o_ids.append(key_) return bow, embed, o_sents, o_pos1, o_pos2, o_e1, o_e2, o_ids, dict_origin_data
def main(exp:ExpDesc, timed_run_params:TimedRunParams, tool_run_params:ToolRunParams): run_stats, tool_rc = timed_run.main(timed_run_params, tool_run_params) tool_log_str = readfile(tool_run_params.log_file) if tool_rc in (REAL_RC, UNREAL_RC): run_result = extract_data(readfile(tool_run_params.output_file) if tool_rc==REAL_RC else None, tool_log_str, tool_rc) elif tool_rc == TIMEOUT_RC: run_result = RunResult(None, None, None, TIMEOUT_STR, None) else: run_result = RunResult(None, None, None, FAIL_STR, None) adhoc_data = extract_adhoc_data(tool_log_str, adhoc_fields) run_result.total_time_sec = run_stats.wall_time_sec run_result.memory_mb = run_stats.virt_mem_mb upload_run(exp, timed_run_params, tool_run_params, run_result, adhoc_data)
def get_capt_hints(fname): hints = dict() content = utils.readfile(fname) for line in content: if not line: continue fields = line.split('|') hints[fields[0]] = fields[1] return hints
def get_capt_labels(fname): labels = dict() content = utils.readfile(fname) for line in content: if not line: continue fields = line.split('|') labels[fields[0]] = fields[1] return labels
def create_aux_files(dsl, dest): for aux in dsl.aux_files: if not os.path.exists(aux.source): continue if aux.get('template'): f = template(aux.source) else: f = readfile(aux.source) name = aux.name setfile('${dest}/${name}', f)
def update_p(request, name, iteration): if request.method == 'GET' and request.GET['tp'] and request.GET['i'] and request.GET['delta']: tp = request.GET['tp'] i = int(request.GET['i']) delta = float(request.GET['delta']) iteration_path = os.path.join(config.CATEGORY_BASEDIR, name, iteration) if os.path.isfile(iteration_path) and can_update(name, iteration): instances, patterns = utils.readfile(iteration_path) utils.ppg(tp, i, delta, instances, patterns) utils.write2file(iteration_path, instances, patterns) return HttpResponse('') return HttpResponseNotFound('iteration %s/%s not exist!' % (name, iteration))
def get_capt_set(fname): capt_sets = [] content = utils.readfile(fname) for line in content: if not line: continue fields = line.split('|') setformat = fields[0].split('_')[:3] capt_sets.append("_".join(setformat)) return capt_sets
def update_p(request, name, iteration): if request.method == 'GET' and request.GET['tp'] and request.GET[ 'i'] and request.GET['delta']: tp = request.GET['tp'] i = int(request.GET['i']) delta = float(request.GET['delta']) iteration_path = os.path.join(config.CATEGORY_BASEDIR, name, iteration) if os.path.isfile(iteration_path) and can_update(name, iteration): instances, patterns = utils.readfile(iteration_path) utils.ppg(tp, i, delta, instances, patterns) utils.write2file(iteration_path, instances, patterns) return HttpResponse('') return HttpResponseNotFound('iteration %s/%s not exist!' % (name, iteration))
def getRawData(token): """获取原始文件 Arguments: - `token`: """ fpath = BASE + token + SUFFIX if os.path.exists(fpath): data = utils.readfile( fpath ) else: data = '' data = re.sub( '\/\*[\s\S]*?\*\/' , '' , data ) return data
def get_all_features(fname): features = [] content = utils.readfile(fname) for line in content: fields = line.split(',') # remove captchaset name fields = fields[1:] for field in fields: if not field: continue # add to features if field not in features: features.append(field) return sorted(features)
def main(args): # parse argument labels = args.labels servtags = args.servtags showname = False feature_dict = dict() # get all possible features features = get_all_features(servtags) for feat in sorted(features): feature_dict[feat] = 0 # get set label label_dict = get_capt_labels(labels) # construct bitmap content = utils.readfile(servtags) # print header generate_header(showname, feature_dict, label_dict) for line in content: if not line: continue fields = line.split(',') set_name = fields[0] tags = fields[1:] if set_name not in label_dict.keys(): continue bitmap = get_feature_bitmap(copy.deepcopy(feature_dict), tags) try: if showname: sys.stdout.write(set_name + ",") sys.stdout.write(",") #print bitmap for bitm in bitmap: sys.stdout.write(str(bitm)) sys.stdout.write(",") sys.stdout.write(label_dict[set_name]) print except Exception as err: pass
def build_js(self,src,dst,base_dir): ''' handle one js src to dst 合并、替换、加时间戳并按需压缩。 ''' js = os.path.relpath(src,base_dir) subprocess.call( 'node ' + RJS_PATH +' -o name=' + js[0:-3] + ' out='+ dst + ' optimize=none baseUrl='\ + base_dir , shell = True) #repalce content = utils.readfile(dst) content = replace(content,self._target) utils.writefile(dst,content) if C('js_ascii_only'): subprocess.call( 'node ' + RPL_PATH +' '+dst+' '+dst,shell = True) if self._compress: subprocess.call( 'java -jar ' + YC_PATH + ' --type js --charset ' + C('encoding') + ' ' + dst + ' -o ' + dst , shell = True )
def _replace(self): ''' 替换所有文本的变量 ''' files = utils.FileSearcher(r'.+',self._build_dir).search() for f in files: f = os.path.join(self._build_dir,f) if not utils.isBinary(f): try: content = utils.readfile(f) content = replace(content,self._target) utils.writefile(f,content) except Exception,e: if self._force: log.error('[replace][%s]%s'%(f,e)) else: e
def play(key): real = os.path.splitext(key)[0] try: if key[-4:] == '.key': meta = readkey(real) r = Response(binascii.unhexlify(meta['key']), mimetype='application/octet-stream') r.headers.add('Access-Control-Allow-Origin', '*') return r meta = readfile(real) if key[-5:] == '.m3u8': r = Response(meta['raw'], mimetype='application/vnd.apple.mpegurl') r.headers.add('Access-Control-Allow-Origin', '*') return r return render_template('play.html', meta=meta) except: return jsonify({'err': 1, 'message': 'File does not exist'})
def compileCss(filepath): if not os.path.exists(filepath): return False css = utils.readfile(filepath) #@todo:正则有问题,识别url("data:image/png,base64...")之类带引号的有[email protected] IMG_TOKEN = 'url\([\'"]?(?!data:image|about:blank|http|https)(.*?)[\'"]?\)' #[email protected]:忽略base64图片和about:blank iters = re.finditer(IMG_TOKEN, css) for i in reversed(list(iters)): imgpath = i.group(1) imgpath = compileCommon(imgpath, 'local', True) #内部可能有替换的变量 if not conf.getConfig().get('disableAutoTimestamp' ): #[email protected]:为什么http开头的不给加时间戳 css = css[0:i.end(0) - 1] + '?t=' + getFileTimeStamp( imgpath, filepath) + css[i.end(0) - 1:] #[email protected]:已经带?的做过识别么? return css
def compileCss(filepath): if not os.path.exists(filepath): return False css = utils.readfile(filepath) # @todo:正则有问题,识别url("data:image/png,base64...")之类带引号的有[email protected] IMG_TOKEN = ( "url\(['\"]?(?!data:image|about:blank|http|https)(.*?)['\"]?\)" ) # [email protected]:忽略base64图片和about:blank iters = re.finditer(IMG_TOKEN, css) for i in reversed(list(iters)): imgpath = i.group(1) imgpath = compileCommon(imgpath, "local", True) # 内部可能有替换的变量 if not conf.getConfig().get("disableAutoTimestamp"): # [email protected]:为什么http开头的不给加时间戳 css = ( css[0 : i.end(0) - 1] + "?t=" + getFileTimeStamp(imgpath, filepath) + css[i.end(0) - 1 :] ) # [email protected]:已经带?的做过识别么? return css
def getData(self,including_deps = True): data = {} if C('disable_deps_search') or not including_deps: deps = [self.__token+'.'+C('template_ext')] else: #复制 deps = self.__deps[0:] deps.reverse() deps.insert(len(deps),self.__token+".json") deps.insert(0,"_ursa.json") for dep in deps: try: json_filepath = utils.abspath(os.path.join(C('data_dir'),re.sub(r'\.%s$'%C('template_ext'),'.json',dep))) content = utils.readfile(json_filepath) content = re.sub('\/\*[\s\S]*?\*\/','',content) json_data = json.loads(content) data.update(json_data) except Exception, e: e#log.warn('[getdata]%s:%s'%(json_filepath,e))
def getFileTimeStamp(fpath, parentpath=''): """为文件加上时间戳并返回 Arguments: - `fpath`: """ if fpath.find('/') == 0: fpath = fpath[1:] fpath2 = os.path.join(conf.getConfig()['path'], 'build', fpath) if not os.path.exists(fpath2) and parentpath: parentpath = parentpath.split('/') parentpath.pop() fpath2 = '/'.join(parentpath) + '/' + fpath if os.path.exists(fpath2): f = utils.readfile(fpath2, 'rb') m = hashlib.md5() m.update(f) md5 = utils.md5toInt(m.hexdigest()) return md5 return ''
def getFileTimeStamp(fpath, parentpath=""): """为文件加上时间戳并返回 Arguments: - `fpath`: """ if fpath.find("/") == 0: fpath = fpath[1:] fpath2 = os.path.join(conf.getConfig()["path"], "build", fpath) if not os.path.exists(fpath2) and parentpath: parentpath = parentpath.split("/") parentpath.pop() fpath2 = "/".join(parentpath) + "/" + fpath if os.path.exists(fpath2): f = utils.readfile(fpath2, "rb") m = hashlib.md5() m.update(f) md5 = utils.md5toInt(m.hexdigest()) return md5 return ""
def _search(self,tpl): ''' 递归搜索 ''' try: abspath = utils.abspath(os.path.join(C('template_dir'),tpl)) content = utils.readfile(abspath) iters = re.finditer(self._pattern,content) for i in reversed(list(iters)): tpl = utils.filterRelPath(i.group(3)) if C('ignore_parents') and tpl.endswith('parent.'+C('template_ext')): continue if self._history.get(tpl) is None: self._result.append(tpl) self._history[tpl] = 1 if 'include' == i.group(1): self._include_result.append(tpl) self._search(tpl) except Exception, e: log.error('[deps]%s'%e)
def save(data, filename, seperator1, seperator2, sd): publickey = "" privatekey = "" values = {} if not os.path.exists(filename): f = open(filename, "w+") with open(filename) as f: file_data = f.read() if len(file_data) > 0: privatekey, publickey, values = readfile(file_data, seperator1, seperator2) else: privatekey, publickey = rsakeys() values[sv] = sd values[data["username"] + data["web"] + data["type"]] = data result = "" for i in values: res = json.dumps(values[i]) res = encrypt(publickey, res.encode()).decode("utf-8") result += str(len(str(len(res)))) + str(len(res)) result += res private_string = privatekey.exportKey("PEM").decode("utf-8") public_string = publickey.exportKey("PEM").decode("utf-8") full_string = private_string + result + public_string full_string = list(full_string) r1 = len(private_string) r2 = len(public_string) n = len(full_string) full_string.insert(seperator1 % n, r1) full_string.insert(seperator2 % n, r2) full_string.insert(0, len(str(r1))) full_string.insert(len(full_string), len(str(r2))) full_string = ''.join(map(str, full_string)) full = ','.join(format(ord(x), 'b') for x in full_string).split(",") n = len(full) file1 = open(filename, "w") full = swaper(0, n, full, 1, seperator1, seperator2) file1.write(','.join(map(str, full)))
def minimize(filename, station_dict): minimum_prec, minimum = 15000000000, 10000000000 trolleybuses = utils.readfile(filename) T_prime, alpha = {}, {} troleibus =[] while(minimum_prec > minimum): troleibus=[] for key in trolleybuses.keys(): T_prime[key] = utils.add_one(trolleybuses, trolleybuses[key]) for key in T_prime.keys(): T_prime[key].sort() alpha[key] = utils.calc_sum(T_prime[key]) minimum_key = min(alpha, key=alpha.get) minimum_prec = minimum minimum = alpha[minimum_key] for key in station_dict: for bus in station_dict[key]: if(minimum_key == bus): station_dict[key][bus] = [x+1 for x in station_dict[key][bus]] return station_dict
def __init__(self, exp_desc:ExpDesc, timed_run_params:TimedRunParams, tool_run_params:ToolRunParams, run_result:RunResult, *args, **kwargs): super().__init__(*args, **kwargs) self.total_time_sec = run_result.total_time_sec self.circuit_size = run_result.circuit_size self.memory_mb = run_result.memory_mb self.is_realizable = run_result.is_realizable self.model = run_result.model self.input_file = tool_run_params.input_file self.logs = readfile(tool_run_params.log_file) if tool_run_params.log_file else '' self.tool_params = tool_run_params.params self.exp = exp_desc.exp_name self.commit = exp_desc.commit self.hardware = exp_desc.hardware self.datetime = exp_desc.datetime self.note = exp_desc.note self.time_limit_sec = timed_run_params.time_limit_sec self.memory_limit_mb = timed_run_params.memory_limit_mb
from jaeschke import t_2 from utils import readfile bases = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53] primes = readfile("primes/primes_1m.txt") if __name__ == "__main__": print(3, 10) t_2(bases[:3], 10**10, primes)
source_dataset = ImageFolder('real_or_drawing/train_data', transform=source_transform) # (32, 32, 3) target_dataset = ImageFolder('real_or_drawing/test_data', transform=target_transform) test_dataset = ImageFolder('real_or_drawing/test_data', transform=test_transform) source_dataloader = DataLoader(source_dataset, batch_size=32, shuffle=True) target_dataloader = DataLoader(target_dataset, batch_size=32, shuffle=True) test_dataloader = DataLoader(test_dataset, batch_size=128, shuffle=False) device = torch.device( f'cuda:{args.cuda}' if torch.cuda.is_available() else 'cpu') arch = f'arch/{args.arch}' if args.do_preprocess: target_datas, target_labels = readfile('real_or_drawing/test_data/0') print('[*] Saving datas...') with open('preprocessed/target_datas.pkl', 'wb') as f: pickle.dump(target_datas, f) if args.do_train: trainer = Trainer(arch, device) for epoch in range(args.max_epoch): trainer.run_epoch(epoch, source_dataloader, target_dataloader, lamb=0.1) accept_indice, sudo_labels, total_labels = get_sudo_label( arch, device, test_dataloader, trainer.feature_extractor, trainer.label_predictor)
from cnn import VGG13, VGG16, VGG19 from mobileNet import StudentNet from utils import readfile, set_seed from dataset import ImgDataset from train import training, deep_mutual_learning import torch.nn.functional as F import torchvision.models as models # Set Random seed SEED = 0 set_seed(SEED) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") workspace_dir = sys.argv[1] print("Reading data") train_x, train_y, train_label_list = readfile( os.path.join(workspace_dir, "training"), True) #print("Size of training data = {}".format(len(train_x))) val_x, val_y, val_label_list = readfile( os.path.join(workspace_dir, "validation"), True) #print("Size of validation data = {}".format(len(val_x))) now_train_count = 0 now_val_count = 0 #print(train_label_list) for i in range(11): if i == 0: train_X = np.concatenate( (train_x[:int(train_label_list[i] * 0.8), :, :, :], val_x[:val_label_list[i], :, :, :])) val_X = train_x[int(train_label_list[i] * 0.8):train_label_list[i], :, :, :]
def uploadfile(username, agencydata, comments, md5sum, sizeoffile, bounds): ## todo: cache if model.Message.all().filter('md5sum =', md5sum).count() > 0: raise UploadError('This file has previously been uploaded') ## todo: cache if model.SkipMd5.all().filter('md5sum =', md5sum).count() > 0: raise UploadError('This file has previously been uploaded') raw_agencies = utils.readfile(agencydata) if not raw_agencies: raise UploadError( "zip file did not contain any valid agencies in agency.txt.") ## save our msg new_message = model.Message(user=username, content=comments) new_message.hasFile = True memcache.delete('Message.recent') # new_message.filename = filename new_message.md5sum = md5sum new_message.size = sizeoffile new_message.max_lat = None new_message.max_lng = None new_message.min_lat = None new_message.min_lng = None if bounds: bounds_list = bounds.split("|") try: new_message.max_lat = float(bounds_list[0]) new_message.max_lng = float(bounds_list[1]) new_message.min_lat = float(bounds_list[2]) new_message.min_lng = float(bounds_list[3]) except ValueError: logging.error('failed to set bounds from %s' % bounds) new_message.put() d = datetime.datetime.now() datestr = d.strftime('%Y%m%d_%H%M') seen_agencies = [] for ag in raw_agencies: ## get from the db ## lookup by url first a = None if ag.get('agency_url', '').strip(): ## try to get via url first as it's more unique url = ag['agency_url'].strip() try: # TODO: use urlnorm url_parsed = urlparse.urlparse(url) if not url_parsed.path: url += '/' except: logging.exception('unable to parse url') a = model.Agency.all().filter('url =', url).get() if not a: slug = model.slugify(ag['agency_name'].strip()) s = utils.lookup_agency_alias(slug) if s: slug = s a = memcache.get('Agency.slug.%s' % slug) if not a: a = model.Agency.all().filter('slug =', slug).get() if a: a.messagecount += 1 a.lastupdate = datetime.datetime.now() a.put() memcache.set('Agency.slug.%s' % a.slug, a) if not a: a = model.Agency() a.name = ag['agency_name'].strip() a.url = ag.get('agency_url', '') a.messagecount = 1 a.put() memcache.delete('Agency.recent') utils.incrAgencyCount() if len(raw_agencies) == 1: new_message.filename = '%s_%s.zip' % (a.slug, datestr) new_message.put() # some zip files have the same url several times; only capture the first time that url is used if a in seen_agencies: continue seen_agencies.append(a) ma = model.MessageAgency() ma.agency = a ma.message = new_message ma.hasFile = True ma.put() memcache.delete('Agency.all') # because it has the cached last-update if not new_message.filename: new_message.filename = '%s_%s.zip' % (username.nickname(), datestr) new_message.put() # TODO: can we even hit this, since upload should only be called at a rate of once a minute anyway? recentFiles = model.Message.all().filter('hasFile =', True).filter( 'date >=', d.replace(second=0, microsecond=0)).count() if recentFiles > 1: # note we already saved *this* filename new_message.filename = new_message.filename.replace( '.zip', '_%d.zip' % recentFiles) new_message.put() ## send email to user ? return new_message.filename
#!/usr/bin/env python from utils import readfile import matplotlib.pyplot as plt import numpy as np import sys if __name__ == "__main__": x = readfile(sys.argv[1]) nbins = min(np.size(x), 100) plt.figure() plt.subplot(211) plt.hist(x.flatten(), nbins) plt.title("Value Histogram") plt.xlabel("Value") plt.ylabel("Frequency") plt.subplot(212) ind1d = np.argsort(-x, axis=None)[:2000] ind2d = np.unravel_index(ind1d, x.shape) plt.plot(ind2d[1], ind2d[0], 'o') plt.gca().invert_yaxis() plt.show()
def main(args): # parse argument labels = args.labels servtags = args.servtags hints = args.hints # make output folder OUTDIR = "test" if not os.path.exists(OUTDIR): os.makedirs(OUTDIR) # get all possible features feature_dict = dict() features = get_all_features(servtags) for feat in sorted(features): feature_dict[feat] = 0 # get captcha set capt_sets = get_capt_set(labels) # get set label label_dict = get_capt_labels(labels) # construct bitmap content = utils.readfile(servtags) # constuct hint dict hints = get_capt_hints(hints) servtags = dict() for line in content: if not line: continue fields = line.split(',') set_name = fields[0] tags = fields[1:] servtags[set_name] = tags for cset in capt_sets: filept = open(os.path.join(OUTDIR, cset + ".arff"), 'w') hint = hints[cset] generate_header(cset, hint, feature_dict, filept) for i in range(10): bitmap = get_feature_bitmap(copy.deepcopy(feature_dict), servtags[cset + "_" + str(i)]) try: bmstring = ",".join( str(bm) for bm in bitmap) if hint == label_dict[cset + "_" + str(i)]: bmstring += ",1" else: bmstring += ",0" bmstring += "\n" filept.write(bmstring) except Exception as err: print err filept.close()
objective.backward() # derive gradient optimizer.step() # update parameters print("raw maximized layer_activations imgs:",x.shape,"select:",0,"in",x.shape[0]) filter_visualization = x.detach().cpu().squeeze()[0] hook_handle.remove() # reminber to rm it, or it exsits forever in every time forwarding return filter_activations, filter_visualization if __name__ == "__main__": workspace_dir = sys.argv[1] #'/home/shannon/Downloads/food-11' model_filename = sys.argv[2] output_dir = sys.argv[3] cnnids = [7,14,14,14,24] filterids = [0,0,1,2,0] print("Reading data") train_x, train_y = readfile(os.path.join(workspace_dir, "training"), True) print("Size of training data = {}".format(len(train_x))) train_set = ImgDataset(train_x, train_y, test_transform) print("Loading model") model = Classifier().cuda() model.load_state_dict(torch.load(model_filename)) # showing filters from assigned indices image img_indices = [800,1602,2001,3201,4001,4800,5600,7000,7400,8003,8801] images, labels = train_set.getbatch(img_indices) for i, (cnnid,filterid) in enumerate(zip(cnnids,filterids)): filter_activations, filter_visualization = filter_explaination(images, model, cnnid=cnnid, filterid=filterid, iteration=100, lr=0.1) print(images.shape) print(filter_activations.shape)
menu = Menu() menu.title = 'IT' menu.page_id = it_page.id # NOTE: As it refer to gaming_page.id, we need to commit gaming_page first menu.order = 1 db.session.add(menu) db.session.commit() # All initial posts post = Page() post.title = 'Doom VFR datang!' post.excerpt = """ Pertama kali saya menguji headset VR modern, saya bermain Doom.Demo tahun 2013 PAX West saya datang dari eksekutif Oculus Brendan Iribe, yang memasang headset VR yang terpasang di dompet, sebelum menyalakan versi modifikasi Doom 3. Hampir seketika, saya memuji perendaman itu. Aku melepaskan dan menumbuhkan kemampuanku untuk segera mengalihkan kepalaku ke barisan tembakan setan. Saya menghargai trik pencahayaan dan perspektif yang digunakan untuk menyampaikan betapa banyak kekacauan yang terjadi di sekitarku. Tidak ada yang seperti saat itu. """ post.content = readfile( os.path.join(os.path.dirname(os.path.realpath(__file__)), 'doomfvr.txt')) post.category = 'gaming' post.tag = 'doom,vr,feature' db.session.add(post) post = Page() post.title = 'Bug memalukan pada OSX Sierra' post.excerpt = """ Kekacauan yang luar biasa! """ post.content = \ ''' <h1>Siapa yang salah ini?</h1> <p>Masa' bisa begitu saja error?</p> '''
from numpy.linalg import inv from utils import readfile class LinearRegression: def __init__(self): self.w = None def fit(self, x, y): X = np.matrix(np.insert(x, 0, [1], axis=1)) Y = np.matrix(y) W = inv(X.transpose() * X) * X.transpose() * Y.transpose() self.w = np.asarray(W).reshape(-1) def predict(self, x): return -1 if np.dot(np.insert(x, 0, [1]), self.w) < 0 else 1 if __name__ == '__main__': train = readfile('train.dat') reg = LinearRegression() reg.fit(*train) test = readfile('test.dat') errors = 0 for i in range(len(test[0])): if reg.predict(test[0][i]) * test[1][i] < 0: errors += 1 print(errors / len(test[0]))
processor_valid = True else: gpu_rg = r'cuda\:(\d{1,2})' m = re.search(gpu_rg, p, flags=re.I) if m: gpu_num = int(m.group(1)) if gpu_num <= device_count() and gpu_num > 0: processor_valid = True if processor_valid is False: exit("Processor type is invalid - only 'cuda' and 'cpu' are valid device types. Only up to cuda:%d are valid" % device_count()) print("Using {}.".format(args.processor)) print("Loading target language from {} and source language from {}.".format(args.targetfile, args.sourcefile)) target, source = readfile(args.targetfile, args.sourcefile) print("Loading model from {}.".format(args.modelfile)) pre_trained_model = load_gensim_model(args.modelfile) print("Removing words not found in the model.") target_data, source_data = remove_words(target, source, pre_trained_model) print("Splitting data into training and testing sets, {}/{}.".format(round(100-(test_size*100)), round(test_size*100))) target_train, target_test, source_train, source_test = split_data(target_data, source_data, test_size) if b >= len(target_train): exit("Error: training batch must be lower than training data.") # train trigram model
# coding: utf8 import pandas as pd import utils loaded_df = utils.readfile("data/", "housing.csv") print(loaded_df.head()) loaded_df2 = utils.readfile( "C:/Users/Hanna/Documents/DSH/GitHub/Predictive/temperature/", "monthly_json.json") print(loaded_df2.head()) if __name__ == '__main__': # bei Direktaufruf soll das ausgeführt werden, nicht beim Import! print('Hello')
from utils import readpretrain from tag import Tag #from mask import Mask trn_file = "train.input" dev_file = "dev.input" tst_file = "test.input" pretrain_file = "sskip.100.vectors" tag_info_file = "tag.info" #trn_file = "train.input.part" #dev_file = "dev.input.part" #tst_file = "test.input.part" #pretrain_file = "sskip.100.vectors.part" UNK = "<UNK>" trn_data = readfile(trn_file) word_to_ix = {UNK: 0} lemma_to_ix = {UNK: 0} ix_to_lemma = [UNK] for sentence, _, lemmas, tags in trn_data: for word in sentence: if word not in word_to_ix: word_to_ix[word] = len(word_to_ix) for lemma in lemmas: if lemma not in lemma_to_ix: lemma_to_ix[lemma] = len(lemma_to_ix) ix_to_lemma.append(lemma) ############################################# ## tags tags_info = Tag(tag_info_file, ix_to_lemma) SOS = tags_info.SOS
parser.add_option("-i", "--ip",action="store", type="string", dest="ip",help="ip address option") parser.add_option("-u", "--url",action="store", type="string", dest="url",help="url option") parser.add_option("-j", "--json",action="store", type="string", dest="json",help="config json option") parser.add_option("-c", action="store_true", dest="iscreate",help="create option") parser.add_option("-m", action="store_true", dest="ismonitor",help="monitor option") options, args = parser.parse_args() logger.info('iscreate=%s' % (options.iscreate)) logger.info('ismonitor=%s' % (options.ismonitor)) isUsingLinux = utils.isUsingLinux or utils.isBeingDebugged if (isUsingLinux): __config__ = utils.SmartObject() logger.debug('(+++) options.json=%s' % (options.json)) if (options.json) and (os.path.exists(options.json)): json = ''.join(utils.readfile(options.json)) __config__ = utils.SmartObject(args=simplejson.loads(json)) logger.debug('(+++) __config__=%s' % (__config__)) else: logger.warning('Missing config.json from the -j option so using default services.') services = ["CPU Load","Current Users","PING","SSH","Total Processes","Zombie Processes"] if (len(__config__.keys()) > 1): services = [k for k in __config__.keys() if (k not in ['DISKS','__dict__','PARTITIONS'])] logger.debug('(+++) services=%s' % (services)) if (options.url): if (options.host): if (options.ip): if (options.iscreate):
def videos(skip): try: skip = (int(skip) - 1) * 50 except: skip = 0 return jsonify({'err': 0, 'data': readfile(skip)})
def mount_packages(): on_abort(umount_packages) jailname = readfile(e('${OBJDIR}/jailname')) sh('mkdir -p ${INSTUFS_DESTDIR}/usr/ports/packages') sh('mount -t nullfs ${OBJDIR}/ports/packages/${jailname}-p ${INSTUFS_DESTDIR}/usr/ports/packages')
def mount_packages(): on_abort(umount_packages) jailname = readfile(e('${OBJDIR}/jailname')) sh('mkdir -p ${INSTUFS_DESTDIR}/usr/ports/packages') sh('mount -t nullfs ${OBJDIR}/ports/data/packages/${jailname}-p ${INSTUFS_DESTDIR}/usr/ports/packages' )