def saveFiles(self): for filename in self.tableDictionary.keys(): dictFile = open(self.tablePath + os.sep + filename,"w") for key in self.tableDictionary[filename].keys(): mkey,columnname = FileUtils.keyValuegetter(key) dictFile.write(FileUtils.keyValueMaker(mkey,columnname,self.tableDictionary[filename][key])) dictFile.write("\n") dictFile.close()
def __init__(self): self.mainWindow = MainFrame(None) self.save_path = os.path.join(os.environ['HOME'], 'WebcamPhotos') fu.checkWriteOnDirectory(self.save_path) self.mainWindow.btnPhoto.Bind(wx.EVT_BUTTON, self.onClick) self.mainWindow.Bind(wx.EVT_CLOSE, self.Close, self.mainWindow)
def split_file_by(filename, folder, offset=0, duration=86400): """ split the file for applying the forecasting algorithm """ src = ds.FileDataSource(filename, None) cut = fu.PeriodicCutProcess(int(duration), int(offset)) src.load() src.data = cut.batch_process(src.data) spl = fu.Splitter(src.data) spl.splitFiles(folder, int(duration), int(offset))
def loadFile(self,filepath): filelines = open(self.tablePath + os.sep + filepath,'r').readlines() singleDict = {} for line in filelines: #column type can also be incorporated here line = line.rstrip() if(len(line) > 0): key,columnname,value = FileUtils.keyValuegetter(line) singleDict[FileUtils.keyMaker(key,columnname)] = value self.tableDictionary[filepath] = singleDict
def comp_make(self, callback=None): '''Makefile compile. Args: callback (function): Callback of return_future. Returns: None ''' def _done_cb(task_id, stat): '''Done callback. Args: task_id (int): Task ID. stat (dict): Task result. Returns: None ''' callback((stat['detect_error'], '')) make_path = self.chal_path + '/compile' FileUtils.copydir(self.res_path + '/make', make_path) with StackContext(Privilege.fileaccess): shutil.copyfile(self.code_path, make_path + '/main.cpp', \ follow_symlinks=False) FileUtils.setperm(make_path, self.compile_uid, self.compile_gid) with StackContext(Privilege.fullaccess): os.chmod(make_path, mode=0o770) task_id = PyExt.create_task('/usr/bin/make', \ [], \ [ 'PATH=/usr/bin:/bin', 'TMPDIR=/home/%d/compile'%self.uniqid, 'OUT=./a.out', ], \ { 0: StdChal.null_fd, 1: StdChal.null_fd, 2: StdChal.null_fd, }, \ '/home/%d/compile'%self.uniqid, 'container/standard', \ self.compile_uid, self.compile_gid, 60000, 1024 * 1024 * 1024, \ PyExt.RESTRICT_LEVEL_LOW) if task_id is None: callback(PyExt.DETECT_INTERNALERR) else: PyExt.start_task(task_id, _done_cb)
def comp_make(self, callback=None): '''Makefile compile. Args: callback (function): Callback of return_future. Returns: None ''' def _done_cb(task_id, stat): '''Done callback. Args: task_id (int): Task ID. stat (dict): Task result. Returns: None ''' callback((stat['detect_error'], '')) make_path = self.chal_path + '/compile' FileUtils.copydir(self.res_path + '/make', make_path) with StackContext(Privilege.fileaccess): shutil.copyfile(self.code_path, make_path + '/main.cpp', \ follow_symlinks=False) FileUtils.setperm(make_path, self.compile_uid, self.compile_gid) with StackContext(Privilege.fullaccess): os.chmod(make_path, mode=0o770) task_id = PyExt.create_task('/usr/bin/make', \ [], \ [ 'PATH=/usr/bin:/bin', 'TMPDIR=/home/%d/compile'%self.uniqid, 'OUT=./a.out', ], \ { 0: StdChal.null_fd, 1: StdChal.null_fd, 2: StdChal.null_fd, }, \ '/home/%d/compile'%self.uniqid, 'container/standard', \ self.compile_uid, self.compile_gid, 60000, 1024 * 1024 * 1024, \ PyExt.RESTRICT_LEVEL_LOW) if task_id is None: callback((PyExt.DETECT_INTERNALERR, '')) else: PyExt.start_task(task_id, _done_cb)
def insertKey(self, key, columns): keyDict = self.tableObject.getDictionaryFromKey(key) for column in columns: #using function since lower data structure can change keyDict.update({ FileUtils.keyMaker(key, column.columnName): column.columnValue })
def getValuedFromKey(self, key, columns): keyDict = self.tableObject.getDictionaryFromKey(key) row = Row(key) for column in columns: row.addcolumn( Column( column.columnName, keyDict.get(FileUtils.keyMaker(key, column.columnName), None))) return row
def GET(self): # beg_date = "1997-05-23 03:14:15" # end_date = "1997-05-23 03:14:15" # channel_id = 'daikun' # conn = DBUtils.get_connection() # comment_list = Comment.Comment.query_by_period_tuples(conn, beg_date, end_date, channel_id) # DBUtils.release_connection(conn) return None return FileUtils.generate_comment_xml(comment_list)
def get_distance(file0, file1, rate): dat0 = ds.FileDataSource(file0, None) dat1 = ds.FileDataSource(file1, None) dat0.load() dat1.load() gdt = fu.Get_Distance() dist = gdt.levenshtein(dat0.data, dat1.data, int(rate)) print "total-distance (d) = %d"%(dist[0]) print "total-time-length (l) = %d"%(dist[1]) print "normalized-distance (d/l) = %f"%(dist[0]*1.0/dist[1])
def prepareTest(self, file): pwd = os.getcwd() #root = pwd + "/Tests/" root = pwd metaModel = MetaModel() fileTxt = file.replace(".sql", ".txt") metaModel.prepareMetaModel(root + "/" + file, root + "log.txt") if not os.path.exists(root + '/temp'): os.makedirs(root + '/temp') smellDetector = SmellDetector(metaModel, root + "/temp/", file) if (os.path.isfile(root + "/temp/" + fileTxt)): os.remove(root + "/temp/" + fileTxt) smellDetector.detectAllDbSmells() return FileUtils.readFileContents(root + "/temp/" + fileTxt)
def POST(self): post = web.input() beg_date = post.get('beg_date') end_date = post.get('end_date') channel_id = post.get('channel_id') for key, val in post.items(): print(key, val) conn = DBUtils.get_connection() comment_list = Comment.Comment.query_by_period_tuples( conn, beg_date, end_date, channel_id) DBUtils.release_connection(conn) return FileUtils.generate_comment_xml(comment_list)
def setKey(key, value): if value == None: print >> sys.stderr, "No value defined for key '" + str(key) + "'" return if key not in Settings.KEY_TYPE: print >> sys.stderr, "Unknown key '" + str(key) + "'" return if Settings.KEY_TYPE[key].get("type") == "file": fullPath = os.path.abspath(value) if not os.path.exists(os.path.abspath(value)): print >> sys.stderr, "No file at '" + fullPath + "'" return if not os.path.isfile(fullPath): print >> sys.stderr, "'" + fullPath + "' is not a file" return expectedMD5 = Settings.KEY_TYPE[key].get("md5") if Settings.KEY_TYPE[key].get("md5") != None: print >> sys.stderr, "Determining MD5 for '" + fullPath + "'" md5 = FileUtils.getFileMd5(fullPath) if md5 != expectedMD5: print >> sys.stderr, "MD5 '" + md5 + "' does not match expected value '" + expectedMD5 + "'" return #print >> sys.stderr, "Defining value '" + fullPath + "' for key '" + key + "'" Settings.setLocal(key, value)
def prepareMetaModel(self, file, logFile): FileUtils.log(logFile, "Processing " + str(file)) with open(file, 'r', errors='ignore') as f: for line in f: MetaModel.processLine(self, line, logFile)
def start(self): '''Start the challenge. Returns: dict: Challenge result. ''' cache_hash = None cache_gid = None # Check if special judge needs to rebuild. if self.judge_typ in ['ioredir']: hashproc = process.Subprocess( \ ['./HashDir.py', self.res_path + '/check'], \ stdout=process.Subprocess.STREAM) dirhash = yield hashproc.stdout.read_until(b'\n') dirhash = int(dirhash.decode('utf-8').rstrip('\n'), 16) ret = StdChal.build_cache_find(self.res_path) if ret is not None and ret[0] == dirhash: cache_hash, cache_gid = ret judge_ioredir = IORedirJudge('container/standard', \ '/cache/%x'%cache_hash) else: cache_hash = dirhash _, cache_gid = StdChal.get_standard_ugid() build_ugid = StdChal.get_standard_ugid() build_relpath = '/cache/%x'%cache_hash build_path = 'container/standard' + build_relpath judge_ioredir = IORedirJudge('container/standard', \ build_relpath) if not (yield judge_ioredir.build(build_ugid, self.res_path)): return [(0, 0, STATUS_ERR)] * len(self.test_list), '' FileUtils.setperm(build_path, \ Privilege.JUDGE_UID, cache_gid, umask=0o750) with StackContext(Privilege.fullaccess): os.chmod(build_path, 0o750) StdChal.build_cache_update(self.res_path, cache_hash, cache_gid) print('StdChal %d built checker %x'%(self.chal_id, cache_hash)) StdChal.build_cache_incref(cache_hash) print('StdChal %d started'%self.chal_id) # Create challenge environment. self.chal_path = 'container/standard/home/%d'%self.uniqid with StackContext(Privilege.fileaccess): os.mkdir(self.chal_path, mode=0o771) try: yield self.prefetch() print('StdChal %d prefetched'%self.chal_id) if self.comp_typ in ['g++', 'clang++']: ret, verdict = yield self.comp_cxx() elif self.comp_typ == 'makefile': ret, verdict = yield self.comp_make() elif self.comp_typ == 'python3': ret, verdict = yield self.comp_python() if ret != PyExt.DETECT_NONE: return [(0, 0, STATUS_CE, verdict)] * len(self.test_list) print('StdChal %d compiled'%self.chal_id) # Prepare test arguments if self.comp_typ == 'python3': exefile_path = self.chal_path \ + '/compile/__pycache__/test.cpython-34.pyc' exe_path = '/usr/bin/python3.4' argv = ['./a.out'] envp = ['HOME=/', 'LANG=en_US.UTF-8'] else: exefile_path = self.chal_path + '/compile/a.out' exe_path = './a.out' argv = [] envp = [] # Prepare judge test_future = [] if self.judge_typ == 'diff': for test in self.test_list: test_future.append(self.judge_diff( exefile_path, exe_path, argv, envp, test['in'], test['ans'], test['timelimit'], test['memlimit'])) elif self.judge_typ == 'ioredir': for test in self.test_list: check_uid, _ = StdChal.get_standard_ugid() test_uid, test_gid = StdChal.get_restrict_ugid() test_future.append(judge_ioredir.judge( \ exefile_path, exe_path, argv, envp, \ (check_uid, cache_gid), \ (test_uid, test_gid), \ '/home/%d/run_%d'%(self.uniqid, test_uid), \ test, self.metadata)) # Emit tests test_result = yield gen.multi(test_future) ret_result = list() for result in test_result: test_pass, data, verdict = result runtime, peakmem, error = data status = STATUS_ERR if error == PyExt.DETECT_NONE: if test_pass is True: status = STATUS_AC else: status = STATUS_WA elif error == PyExt.DETECT_OOM: status = STATUS_MLE elif error == PyExt.DETECT_TIMEOUT \ or error == PyExt.DETECT_FORCETIMEOUT: status = STATUS_TLE elif error == PyExt.DETECT_EXITERR: status = STATUS_RE else: status = STATUS_ERR ret_result.append((runtime, peakmem, status, verdict)) return ret_result finally: if cache_hash is not None: StdChal.build_cache_decref(cache_hash) with StackContext(Privilege.fileaccess): shutil.rmtree(self.chal_path) print('StdChal %d done'%self.chal_id)
def comp_python(self, callback=None): '''Python3.4 compile. Args: callback (function): Callback of return_future. Returns: None ''' def _started_cb(task_id): '''Started callback. Close unused file descriptors after the task is started. Args: task_id (int): Task ID. Returns: None ''' nonlocal errpipe_fd os.close(errpipe_fd) def _done_cb(task_id, stat): '''Done callback. Args: task_id (int): Task ID. stat (dict): Task result. Returns: None ''' nonlocal compile_path with StackContext(Privilege.fileaccess): verfile = open(compile_path + '/verdict.txt', 'r') verdict = verfile.read(140) verfile.close() callback((stat['detect_error'], verdict)) compile_path = self.chal_path + '/compile' with StackContext(Privilege.fileaccess): os.mkdir(compile_path, mode=0o770) shutil.copyfile(self.code_path, compile_path + '/test.py', \ follow_symlinks=False) FileUtils.setperm(compile_path, self.compile_uid, self.compile_gid) with StackContext(Privilege.fileaccess): errpipe_fd = os.open(compile_path + '/verdict.txt', \ os.O_WRONLY | os.O_CREAT | os.O_CLOEXEC, mode=0o440) task_id = PyExt.create_task('/usr/bin/python3.4', \ [ '-m', 'py_compile', './test.py' ], \ [ 'HOME=/home/%d/compile'%self.uniqid, 'LANG=en_US.UTF-8' ], \ { 0: StdChal.null_fd, 1: StdChal.null_fd, 2: errpipe_fd, }, \ '/home/%d/compile'%self.uniqid, 'container/standard', \ self.compile_uid, self.compile_gid, 60000, 1024 * 1024 * 1024, \ PyExt.RESTRICT_LEVEL_LOW) if task_id is None: os.close(errpipe_fd) callback((PyExt.DETECT_INTERNALERR, '')) return PyExt.start_task(task_id, _done_cb, _started_cb)
def build(self, build_ugid, res_path, callback=None): '''Build environment. Args: build_ugid ((int, int)): Build UID/GID. res_path (string): Resource path. callback (function): Callback of return_future. Returns: None ''' def _done_cb(task_id, stat): '''Done callback. Args: task_id (int): Task ID. stat (dict): Task result. Returns: None ''' if stat['detect_error'] == PyExt.DETECT_NONE: callback(True) else: callback(False) build_uid, build_gid = build_ugid # Prepare build environment. FileUtils.copydir(res_path + '/check', self.build_path) FileUtils.setperm(self.build_path, build_uid, build_gid) with StackContext(Privilege.fullaccess): os.chmod(self.build_path, mode=0o770) with StackContext(Privilege.fileaccess): if not os.path.isfile(self.build_path + '/build'): callback(True) return # Make the build file executable. with StackContext(Privilege.fullaccess): os.chmod(self.build_path + '/build', mode=0o770) # Build. task_id = PyExt.create_task(self.build_relpath + '/build', \ [], \ [ 'PATH=/usr/bin:/bin', 'TMPDIR=%s'%self.build_relpath, 'HOME=%s'%self.build_relpath, 'LANG=en_US.UTF-8' ], \ { 0: StdChal.null_fd, 1: StdChal.null_fd, 2: StdChal.null_fd, }, \ self.build_relpath, 'container/standard', \ build_uid, build_gid, 60000, 1024 * 1024 * 1024, \ PyExt.RESTRICT_LEVEL_LOW) if task_id is None: callback(False) else: PyExt.start_task(task_id, _done_cb)
def build(self, build_ugid, res_path, callback=None): '''Build environment. Args: build_ugid ((int, int)): Build UID/GID. res_path (string): Resource path. callback (function): Callback of return_future. Returns: None ''' def _done_cb(task_id, stat): '''Done callback. Args: task_id (int): Task ID. stat (dict): Task result. Returns: None ''' if stat['detect_error'] == PyExt.DETECT_NONE: callback(True) else: callback(False) build_uid, build_gid = build_ugid # Prepare build environment. FileUtils.copydir(res_path + '/check', self.build_path) FileUtils.setperm(self.build_path, build_uid, build_gid) with StackContext(Privilege.fullaccess): os.chmod(self.build_path, mode=0o770) with StackContext(Privilege.fileaccess): if not os.path.isfile(self.build_path + '/build'): callback(True) return # Build. task_id = PyExt.create_task(self.build_relpath + '/build', \ [], \ [ 'PATH=/usr/bin:/bin', 'TMPDIR=%s'%self.build_relpath, 'HOME=%s'%self.build_relpath, 'LANG=en_US.UTF-8' ], \ { 0: StdChal.null_fd, 1: StdChal.null_fd, 2: StdChal.null_fd, }, \ self.build_relpath, 'container/standard', \ build_uid, build_gid, 60000, 1024 * 1024 * 1024, \ PyExt.RESTRICT_LEVEL_LOW) if task_id is None: callback(False) else: PyExt.start_task(task_id, _done_cb)
def comp_python(self, callback=None): '''Python3.4 compile. Args: callback (function): Callback of return_future. Returns: None ''' def _started_cb(task_id): '''Started callback. Close unused file descriptors after the task is started. Args: task_id (int): Task ID. Returns: None ''' nonlocal errpipe_fd os.close(errpipe_fd) def _done_cb(task_id, stat): '''Done callback. Args: task_id (int): Task ID. stat (dict): Task result. Returns: None ''' nonlocal compile_path with StackContext(Privilege.fileaccess): verfile = open(compile_path + '/verdict.txt', 'r') verdict = verfile.read(140) verfile.close() callback((stat['detect_error'], verdict)) compile_path = self.chal_path + '/compile' with StackContext(Privilege.fileaccess): os.mkdir(compile_path, mode=0o770) shutil.copyfile(self.code_path, compile_path + '/test.py', \ follow_symlinks=False) FileUtils.setperm(compile_path, self.compile_uid, self.compile_gid) with StackContext(Privilege.fileaccess): errpipe_fd = os.open(compile_path + '/verdict.txt', \ os.O_WRONLY | os.O_CREAT | os.O_CLOEXEC, mode=0o440) task_id = PyExt.create_task('/usr/bin/python3.4', \ [ '-m', 'py_compile', './test.py' ], \ [ 'HOME=/home/%d/compile'%self.uniqid, 'LANG=en_US.UTF-8' ], \ { 0: StdChal.null_fd, 1: StdChal.null_fd, 2: errpipe_fd, }, \ '/home/%d/compile'%self.uniqid, 'container/standard', \ self.compile_uid, self.compile_gid, 60000, 1024 * 1024 * 1024, \ PyExt.RESTRICT_LEVEL_LOW) if task_id is None: os.close(errpipe_fd) callback(PyExt.DETECT_INTERNALERR) return PyExt.start_task(task_id, _done_cb, _started_cb)
def comp_cxx(self, callback=None): '''GCC, Clang compile. Args: callback (function): Callback of return_future. Returns: None ''' def _started_cb(task_id): '''Started callback. Close unused file descriptors after the task is started. Args: task_id (int): Task ID. Returns: None ''' nonlocal errpipe_fd os.close(errpipe_fd) def _done_cb(task_id, stat): '''Done callback. Args: task_id (int): Task ID. stat (dict): Task result. Returns: None ''' nonlocal compile_path with StackContext(Privilege.fileaccess): verfile = open(compile_path + '/verdict.txt', 'rb') # To fix decoding error. # Force convert the binary string to string temporarily. verdict = ''.join(chr(c) for c in verfile.read(140)) verfile.close() callback((stat['detect_error'], verdict)) compile_path = self.chal_path + '/compile' with StackContext(Privilege.fileaccess): os.mkdir(compile_path, mode=0o770) shutil.copyfile(self.code_path, compile_path + '/test.cpp', \ follow_symlinks=False) FileUtils.setperm(compile_path, self.compile_uid, self.compile_gid) with StackContext(Privilege.fileaccess): errpipe_fd = os.open(compile_path + '/verdict.txt', \ os.O_WRONLY | os.O_CREAT | os.O_CLOEXEC, mode=0o440) if self.comp_typ == 'g++': compiler = '/usr/bin/g++' elif self.comp_typ == 'clang++': compiler = '/usr/bin/clang++' task_id = PyExt.create_task(compiler, \ [ '-O2', '-std=c++14', '-o', './a.out', './test.cpp', ], \ [ 'PATH=/usr/bin:/bin', 'TMPDIR=/home/%d/compile'%self.uniqid, ], \ { 0: StdChal.null_fd, 1: StdChal.null_fd, 2: errpipe_fd, }, \ '/home/%d/compile'%self.uniqid, 'container/standard', \ self.compile_uid, self.compile_gid, 60000, 1024 * 1024 * 1024, \ PyExt.RESTRICT_LEVEL_LOW) if task_id is None: os.close(errpipe_fd) callback((PyExt.DETECT_INTERNALERR, '')) return PyExt.start_task(task_id, _done_cb, _started_cb)
def start(self): '''Start the challenge. Returns: dict: Challenge result. ''' cache_hash = None cache_gid = None # Check if special judge needs to rebuild. if self.judge_typ in ['ioredir']: hashproc = process.Subprocess( \ ['./HashDir.py', self.res_path + '/check'], \ stdout=process.Subprocess.STREAM) dirhash = yield hashproc.stdout.read_until(b'\n') dirhash = int(dirhash.decode('utf-8').rstrip('\n'), 16) ret = StdChal.build_cache_find(self.res_path) if ret is not None and ret[0] == dirhash: cache_hash, cache_gid = ret judge_ioredir = IORedirJudge('container/standard', \ '/cache/%x'%cache_hash) else: cache_hash = dirhash _, cache_gid = StdChal.get_standard_ugid() build_ugid = StdChal.get_standard_ugid() build_relpath = '/cache/%x'%cache_hash build_path = 'container/standard' + build_relpath judge_ioredir = IORedirJudge('container/standard', \ build_relpath) if not (yield judge_ioredir.build(build_ugid, self.res_path)): return [(0, 0, STATUS_ERR)] * len(self.test_list) FileUtils.setperm(build_path, \ Privilege.JUDGE_UID, cache_gid, umask=0o750) with StackContext(Privilege.fullaccess): os.chmod(build_path, 0o750) StdChal.build_cache_update(self.res_path, cache_hash, cache_gid) print('StdChal %d built checker %x'%(self.chal_id, cache_hash)) StdChal.build_cache_incref(cache_hash) print('StdChal %d started'%self.chal_id) # Create challenge environment. self.chal_path = 'container/standard/home/%d'%self.uniqid with StackContext(Privilege.fileaccess): os.mkdir(self.chal_path, mode=0o771) try: yield self.prefetch() print('StdChal %d prefetched'%self.chal_id) if self.comp_typ in ['g++', 'clang++']: ret, verdict = yield self.comp_cxx() elif self.comp_typ == 'makefile': ret, verdict = yield self.comp_make() elif self.comp_typ == 'python3': ret, verdict = yield self.comp_python() if ret != PyExt.DETECT_NONE: return [(0, 0, STATUS_CE)] * len(self.test_list), verdict print('StdChal %d compiled'%self.chal_id) # Prepare test arguments if self.comp_typ == 'python3': exefile_path = self.chal_path \ + '/compile/__pycache__/test.cpython-34.pyc' exe_path = '/usr/bin/python3.4' argv = ['./a.out'] envp = ['HOME=/', 'LANG=en_US.UTF-8'] else: exefile_path = self.chal_path + '/compile/a.out' exe_path = './a.out' argv = [] envp = [] # Prepare judge test_future = [] if self.judge_typ == 'diff': for test in self.test_list: test_future.append(self.judge_diff( exefile_path, exe_path, argv, envp, test['in'], test['ans'], test['timelimit'], test['memlimit'])) elif self.judge_typ == 'ioredir': for test in self.test_list: check_uid, _ = StdChal.get_standard_ugid() test_uid, test_gid = StdChal.get_restrict_ugid() test_future.append(judge_ioredir.judge( \ exefile_path, exe_path, argv, envp, \ (check_uid, cache_gid), \ (test_uid, test_gid), \ '/home/%d/run_%d'%(self.uniqid, test_uid), \ test, self.metadata)) # Emit tests test_result = yield gen.multi(test_future) ret_result = list() for result in test_result: test_pass, data = result runtime, peakmem, error = data status = STATUS_ERR if error == PyExt.DETECT_NONE: if test_pass is True: status = STATUS_AC else: status = STATUS_WA elif error == PyExt.DETECT_OOM: status = STATUS_MLE elif error == PyExt.DETECT_TIMEOUT \ or error == PyExt.DETECT_FORCETIMEOUT: status = STATUS_TLE elif error == PyExt.DETECT_EXITERR: status = STATUS_RE else: status = STATUS_ERR ret_result.append((runtime, peakmem, status)) return ret_result, '' finally: if cache_hash is not None: StdChal.build_cache_decref(cache_hash) with StackContext(Privilege.fileaccess): shutil.rmtree(self.chal_path) print('StdChal %d done'%self.chal_id)