def create_nodes(*args, **kwargs): """Create nodes to the order of magnitude specified by kwargs.magnitude.""" if len(args) > 0 or "mul_fact" in kwargs: if len(args) != 0: suffix = args[0] else: suffix = kwargs["mul_fact"] suffix = str(suffix) # First create the dependency node. f = open(os.path.join(DATA_DIR, "dependency.json")) dependency_node = decode_json(f.read()) CreateNodesInGraph.create_dependency_node( dependency_name=dependency_node["dependency_name"] + "_" + suffix, dependency_path=dependency_node["dependency_path"] + "_" + suffix, ) # Now create two versions for the node. f = open(os.path.join(DATA_DIR, "dependency_version.json")) dependency_version = decode_json(f.read()) # Modify this if the number of versions of a dependency needs to be changed. for i in (0, 2): CreateNodesInGraph.create_dependency_version_node( version="{}_{}_{}".format(dependency_version["version"], suffix, str(i)), dep_name="{}_{}_{}".format( dependency_version["dependency_name"], suffix, str(i)), ) # Create a security event node. This will be equal to the number of dependency nodes. f = open(os.path.join(DATA_DIR, "security_event.json")) security_event = decode_json(f.read()) CreateNodesInGraph.create_security_event_node( event_id=security_event["event_id"], event_type=security_event["event_type"], body=security_event["body"], title=security_event["title"], ) # Create a probable vulnerability node. This will be half of all security event nodes. if i % 2 == 0: CreateNodesInGraph.create_probable_vuln_node("PCVE-20XX-{}".format( str(i))) # Create an identified CVE node for every alternate probable CVE node. This means there'll # be half the number of identified CVE nodes as probale CVE nodes. if i % 4 == 0: f = open(os.path.join(DATA_DIR, "identified_cve.json")) identified_cve = decode_json(f.read()) CreateNodesInGraph.create_reported_cve_node( cve_id=identified_cve["cve_id"], cvss=identified_cve["cvss"], severity=Severity(identified_cve["severity"].lower()), )
def srv_loop(): address = (HOST, PORT) clients = [] sock = start_server(address) while True: try: client, address = sock.accept() except OSError as e: pass else: print(f'Connected with {str(address)}') clients.append(client) finally: w = [] try: r, w, e = select.select([], clients, [], 0) except Exception as e: pass for s_client in w: srv_response = encode_json(response(200)) try: data = s_client.recv(1024) print(decode_json(data)) s_client.send(srv_response) except: clients.remove(s_client)
def onNotification(self, sender, method, data): # pylint: disable=invalid-name ''' Notification event handler for accepting data from add-ons ''' if not method.endswith( 'upnext_data'): # Method looks like Other.upnext_data return data, encoding = decode_json(data) data.update(id='%s_play_action' % sender.replace('.SIGNAL', '')) self.api.addon_data_received(data, encoding=encoding)
def onNotification(self, sender, method, data): # pylint: disable=invalid-name """Notification event handler for accepting data from add-ons""" if not method.endswith('upnext_data'): # Method looks like Other.upnext_data return decoded_data, encoding = decode_json(data) if decoded_data is None: self.log('Received data from sender %s is not JSON: %s' % (sender, data), 2) return decoded_data.update(id='%s_play_action' % sender.replace('.SIGNAL', '')) self.api.addon_data_received(decoded_data, encoding=encoding)
def cast(val): if isinstance(val, basestring): if self.is_base64(val): mimetype = self.get_base64_mimetype(val) base64_string = self.get_base64_content(val) base64_decoded = base64.b64decode(base64_string) base64_string = None bucket = os.environ.get('GCS_BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) filename = utils.get_random_string(12) filename += str(calendar.timegm(time.gmtime())) import cloudstorage as gcs acl = None if self.acl: acl = {'x-goog-acl': self.acl} with gcs.open('/%s/%s' % (bucket, filename), 'w', content_type=mimetype, options=acl) as gcs_file: gcs_file.write(base64_decoded) if self._local: return '%s/%s/%s' % (self.base_path, bucket, filename) access_token, _ = self.get_access_token() medialink_url = '%s/%s/o/%s' % (self.base_path, bucket, filename) medialink = urlfetch.fetch( method=urlfetch.GET, url=medialink_url, headers={ 'Authorization': 'Bearer %s' % access_token, }) if medialink.status_code >= 200 and medialink.status_code < 300: return utils.decode_json(medialink.content)['mediaLink'] return None elif URL_REGEX.findall(val): return val elif LOCAL_URL_REGEX.findall(val): return val else: raise datastore_errors.BadValueError('Expected Base64 string or ' 'URL, got %r.' % val) else: raise datastore_errors.BadValueError('Expected string, got %r.' % val)
def test_decode_json(): assert isinstance(decode_json(b'{"a": 1, "b": "string"}'), dict)
def create_nodes(*args, **kwargs): """Create nodes to the order of magnitude specified by kwargs.magnitude.""" if len(args) > 0 or "mul_fact" in kwargs: if len(args) != 0: suffix = args[0] else: suffix = kwargs["mul_fact"] index = suffix suffix = str(suffix) # First create the dependency node. f = open(os.path.join(DATA_DIR, "dependency.json")) dependency_node = decode_json(f.read()) dependency_node["dependency_name"] += "_" + suffix dependency_node["dependency_path"] += "_" + suffix CreateNodesInGraph.create_dependency_node( dependency_name=dependency_node["dependency_name"], dependency_path=dependency_node["dependency_path"], ) # Now create two versions for the node. f = open(os.path.join(DATA_DIR, "dependency_version.json")) dependency_version = decode_json(f.read()) # Modify this if the number of versions of a dependency needs to be changed. for i in range(0, 2): CreateNodesInGraph.create_dependency_version_node( version="{}_{}".format(dependency_version["version"], str(i)), dep_name="{}_{}".format(dependency_version["dependency_name"], suffix), ) # Since this iterates internally, passing one is enough. dependency_version["dependency_name"] = "{}_{}".format( dependency_version["dependency_name"], suffix) CreateEdges.create_dependency_version_edge(dependency_node, dependency_version) # Create a security event node. This will be equal to the number of dependency nodes. f = open(os.path.join(DATA_DIR, "security_event.json")) security_event = decode_json(f.read()) CreateNodesInGraph.create_security_event_node( event_id=security_event["event_id"], event_type=security_event["event_type"], body=security_event["body"], title=security_event["title"], ) probable_vuln = None # Create a probable vulnerability node. This will be half of all security event nodes. if index % 2 == 0: f = open(os.path.join(DATA_DIR, "probable_vulnerability.json")) probable_vuln = decode_json(f.read()) CreateNodesInGraph.create_probable_vuln_node( probable_vuln["probable_vuln_id"]) # Link this probable vulnerability to the security event node. CreateEdges.create_prob_vuln_sec_event_link( probable_vuln, security_event) # Create an identified CVE node for every alternate probable CVE node. This means there'll # be half the number of identified CVE nodes as probale CVE nodes. if index % 4 == 0: f = open(os.path.join(DATA_DIR, "identified_cve.json")) identified_cve = decode_json(f.read()) CreateNodesInGraph.create_reported_cve_node( cve_id=identified_cve["CVE_ID"], cvss=identified_cve["CVSS"], severity=identified_cve["severity"], ) # Link this identified CVE node to the dependency node with version suffix {1}. dependency_version["version"] = "{}_{}".format( dependency_version["version"], str(1)) CreateEdges.create_reported_cve_dependency_version_link( identified_cve, dependency_version) # Also link it to the probable CVE node. if probable_vuln is not None: CreateEdges.create_probable_reported_cve_link( probable_vuln, identified_cve)
logger.debug("Processing document [%s]", document.get_id()) start = out.tell() out.write("#DOCUMENT\t%s\n" % (document.get_id())) process("HEAD", document, out) process("TEXT", document, out) #print(document["HEAD"].textContent(), document["TEXT"].textContent()) outptr.write("%s\t%d\t%d\n" % (document.get_id(), start, out.tell())) logger.debug("Processed document [%s]", document.get_id()) for collection in args.collections: l = collection.split("#") if args.json and os.path.exists(collection): s = open(collection).read() descriptions = decode_json(s) else: restrict = "" if len(l) == 1: pass elif len(l) == 2: restrict = l[1] collection = l[0] else: raise Exception("Cannot handle collection specification [%s]" % collection) descriptions = collections(collection, restrict) for description in descriptions: collection_id = description["id"] donefilepath = os.path.join(args.outdir, "%s.done" % collection_id)
def import_from_syzoj(url: str, willPublic: bool): """ 从SYZOJ导入题目 参数: url:str SYZOJ题目URL willPublic:int 新题目是否公开 返回 { "code":0, "uuid":'用于websocket的uuid', "message":"" } """ import urllib import tempfile import pathlib import traceback import zipfile import shutil import os import yaml import requests from io import BytesIO from utils import decode_json if not session.get("uid"): return make_response(-1, message="请先登录") user: User = User.by_id(session.get("uid")) if not permission_manager.has_any_permission(user.id, "problem.create", "problem.manage"): return make_response(-1, message="你没有权限执行此操作") try: with requests.get(f"{url}/export") as urlf: data = decode_json(urlf.content.decode())["obj"] print("JSON data: {}".format(data)) import datetime problem = Problem(uploader_id=user.id, title=data["title"], content=data["description"], input_format=data["input_format"], output_format=data["output_format"], hint=data["limit_and_hint"], using_file_io=data["file_io"], input_file_name=data["file_io_input_name"], output_file_name=data["file_io_output_name"], create_time=datetime.datetime.now() ) if willPublic: if not permission_manager.has_any_permission(user.id, "problem.publicize", "problem.manage"): return make_response(-1, message="你没有权限公开题目") problem.public = True problem.example = [] problem.hint = "### 样例\n" + \ data["example"]+"\n\n### Hint\n"+problem.hint time_limit = int(data["time_limit"]) memory_limit = int(data["memory_limit"]) db.session.add(problem) db.session.commit() work_dir = pathlib.PurePath(tempfile.mkdtemp()) with requests.get(f"{url}/testdata/download") as urlf: pack = zipfile.ZipFile(BytesIO(urlf.content)) pack.extractall(work_dir) pack.close() problem_data_dir = pathlib.PurePath( f"{config.UPLOAD_DIR}/{problem.id}") shutil.rmtree(problem_data_dir, ignore_errors=True) shutil.copytree(work_dir, problem_data_dir) shutil.rmtree(work_dir) # 更换新的word_dir work_dir = problem_data_dir for file in filter(lambda x: x.endswith(".lock"), os.listdir(work_dir)): os.remove(work_dir/file) file_list = [] for file in filter(lambda x: not x.endswith(".lock"), os.listdir(work_dir)): with open(work_dir/(file+".lock"), "w") as f: import time last_modified_time = time.time() f.write(str(last_modified_time)) file_list.append({ "name": file, "size": os.path.getsize(work_dir/file), "last_modified_time": last_modified_time }) problem.files = file_list pure_file_list = list(map(lambda x: x["name"], file_list)) for x in pure_file_list: if x.startswith("spj_"): problem.spj_filename = x break auto_generate = True subtasks = [] if os.path.exists(work_dir/"data.yml"): # 存在data.yml with open(work_dir/"data.yml", "r", encoding="utf-8") as f: data_obj = yaml.safe_load(f) # data.yml中钦定spj if "specialJudge" in data_obj: new_spj_filename = work_dir/( "spj_"+data_obj["specialJudge"]["language"]+"."+data_obj["specialJudge"]["fileName"].split(".")[-1]) print(new_spj_filename) print(work_dir/data_obj["specialJudge"]["fileName"]) shutil.move( work_dir/data_obj["specialJudge"]["fileName"], new_spj_filename) problem.spj_filename = new_spj_filename.name if "subtasks" in data_obj: auto_generate = False def make_input(x): return data_obj["inputFile"].replace("#", str(x)) def make_output(x): return data_obj["outputFile"].replace("#", str(x)) for i, subtask in enumerate(data_obj["subtasks"]): print(subtask) subtasks.append({ "name": f"Subtask{i+1}", "score": int(subtask["score"]), "method": subtask["type"], "time_limit": time_limit, "memory_limit": memory_limit, "testcases": [] }) testcases = subtasks[-1]["testcases"] score = subtasks[-1]["score"]//len(subtask["cases"]) for testcase in subtask["cases"]: testcases.append({ "input": make_input(testcase), "output": make_output(testcase), "full_score": score }) testcases[-1]["full_score"] = subtasks[-1]["score"] - \ score*(len(testcases)-1) if auto_generate: # 不存在data.yml,直接生成子任务 input_files = list( filter(lambda x: x.endswith(".in"), pure_file_list)) output_files = list( filter(lambda x: x.endswith(".out") or x.endswith(".ans"), pure_file_list)) if len(input_files) == len(output_files): pass for i, file in enumerate(input_files): pure_file_name = file[:file.rindex(".")] subtasks.append({ "name": f"Subtask{i+1}", "score": 100//len(input_files), "method": "sum", "time_limit": time_limit, "memory_limit": memory_limit, "testcases": [{"full_score": 100//len(input_files), "input": file, "output": f"{pure_file_name}.ans" if f"{pure_file_name}.ans" in output_files else f"{pure_file_name}.out"}], "comment": "" }) diff = 100-sum(map(lambda x: x["score"], subtasks)) subtasks[-1]["score"] += diff subtasks[-1]["testcases"][0]["full_score"] += diff for file in filter(lambda x: x.endswith(".lock"), os.listdir(work_dir)): os.remove(work_dir/file) for file in filter(lambda x: not x.endswith(".lock"), os.listdir(work_dir)): with open(work_dir/(file+".lock"), "w") as f: import time last_modified_time = time.time() f.write(str(last_modified_time)) problem.files = generate_file_list(problem.id) problem.subtasks = subtasks db.session.commit() except Exception: print(traceback.format_exc()) return make_response(-1, message=traceback.format_exc()) return make_response(0, problem_id=problem.id)
def cli_loop(message): address = (HOST, PORT) sock = start_client(address) sock.send(message) data = sock.recv(1024) print(decode_json(data))
def message_for_room(request): data = decode_json(json.loads(request)) room = data['room'] send(json.dumps(json.dumps(data)), room=room)
def on_leave(request): data = decode_json(json.loads(request)) username = data['username'] room = data['room'] leave_room(room) send(username + ' has left the room.', room=room)
def on_join(request): data = decode_json(json.loads(request)) username = data['username'] room = data['room'] join_room(room) send(json.dumps('{0} has entered the room.'.format(username)), room=room)
def remote_judge_submit(data): """ 客户端提交代码 { "problemID":hj2题目ID, "remoteAccountID":"远程用户ID", "code":"用户代码", "language":"用户选择的语言", "loginCaptcha":"登录验证码", "submitCaptcha":"提交验证码", "contestID":"提交为某场比赛,-1设置为不提交比赛", "contestProblemID":"比赛题目ID" } """ if not permission_manager.has_permission(session.get("uid"), "remote_judge.use"): emit("server_response", { "ok": False, "message": "你没有权限这样做" }, room=request.sid) return problem: Problem = Problem.by_id(data["problemID"]) contest_id = data["contestID"] if contest_id != -1: contest: Contest = db.session.query(Contest).filter( Contest.id == contest_id).one_or_none() if not contest: emit("server_response", { "ok": False, "message": "非法比赛ID" }, room=request.sid) return if len(contest.problems) <= data["contestProblemID"]: emit("server_response", { "ok": False, "message": "非法比赛题目ID" }, room=request.sid) return if not contest.running(): emit("server_response", { "ok": False, "message": "比赛未在进行" }, room=request.sid) return problem: Problem = Problem.by_id( contest.problems[data["contestProblemID"]]["id"]) else: if not problem.public and int( session.get("uid") ) != problem.uploader_id and not permission_manager.has_permission( session.get("uid"), "problem.manage"): emit("server_response", { "ok": False, "message": "你没有权限使用此题目" }, room=request.sid) return if len(data["code"]) > config.MAX_CODE_LENGTH: emit("server_response", { "ok": False, "message": "代码过长" }, room=request.sid) return remote_account: RemoteAccount = db.session.query(RemoteAccount).filter( RemoteAccount.account_id == data["remoteAccountID"]).one_or_none() if not remote_account or remote_account.uid != int(session.get("uid")): emit("server_response", { "ok": False, "message": "非法账户ID" }, room=request.sid) return if data["language"] not in config.REMOTE_JUDGE_OJS[ problem.remote_judge_oj]["availableLanguages"]: emit("server_response", { "ok": False, "message": "非法语言" }, room=request.sid) return remote_judge_queue.send_task("judgers.remote.submit", [ problem.remote_judge_oj, decode_json(remote_account.session), remote_account.account_id, problem.remote_problem_id, data["language"], data["code"], data["loginCaptcha"], data["submitCaptcha"], request.sid, remote_account.username, remote_account.password, problem.id, int(session.get("uid")), problem.public, list(reversed(config.TRACK_DELAY_INTERVAL)), data["contestID"], data["contestProblemID"] ]) emit("message", {"message": "提交中..."}, room=request.sid)