def send_query(self): splitpath = urllib.parse.urlsplit(self.path) conditions = {} logging.debug("send_query(): query: %s", splitpath.query) if splitpath.query: # Parse query part into SELECT conditions # Now look at individual key=value pairs for query in splitpath.query.split("&"): query = urllib.parse.unquote_plus(query) # logging.debug("Processing token " + str(query)) for (name, op) in wudb.MyCursor.name_to_operator.items(): if op in query: (key, value) = query.split(op, 1) key = key.strip() value = value.strip() if not name in conditions: conditions[name] = {} # If value is of the form "now(-123)", convert it to a # time stamp of 123 minutes ago r = re.match(r"now\((-?\d+)\)", value) if r: minutes_ago = int(r.group(1)) td = datetime.timedelta(minutes=minutes_ago) value = str(datetime.datetime.utcnow() + td) conditions[name][key] = value break logging.debug("send_query(): conditions: %s", conditions) if self.db_pool: wus = self.db_pool.query(**conditions) else: wus = wudb.WuAccess(self.dbdata).query(**conditions) body = HtmlGen() body.append('<a href="/index.html">Back to index</a>') body.append("<p>Query for conditions = " + str(conditions) + "</p>") if not wus is None and len(wus) > 0: cwd = os.getcwd() body.append(str(len(wus)) + " records match.") keys = wus[0].keys() body.start_table(keys) for wu in wus: body.wu_row(wu, keys, cwd) body.end_table() else: body.append("No records match.") body.finish() self.send_response(200) self.send_header("Content-Type", "text/html") self.send_header("Cache-Control", "no-cache") self.send_header("Content-Length", body.get_len()) self.end_headers() self.send_body(body.__bytes__())
def send_WU(self): splitpath = urllib.parse.urlsplit(self.path) # print(str(splitpath)) clientid_match = self.clientid_pattern.match(splitpath.query) if not clientid_match: return self.send_error(400, "No or malformed client id specified") clientid = clientid_match.group(1) if not self.serving_wus[0]: try: return self.send_error(410, "Distributed computation finished") except socket_error as e: self.log_error("Connection error: %s", str(e)) return if self.db_pool: wu_text = self.db_pool.assign(clientid, timeout_hint=self.timeout_hint) else: wu_text = wudb.WuAccess(self.dbdata).assign( clientid, timeout_hint=self.timeout_hint) if not wu_text: # This flag is to downgrade the logging level. Ugly. self.no_work_available = True try: return self.send_error(404, "No work available") except socket_error as e: self.log_error("Connection error: %s", str(e)) return self.log_message("Sending workunit " + Workunit(wu_text).get_id() + " to client " + clientid) # wu_text = wu.get_wu() try: self.send_response(200) self.send_header("Content-Type", "text/plain") self.send_header("Content-Length", len(wu_text)) self.send_header("Cache-Control", "no-cache") self.end_headers() # FIXME: is ASCII enough for workunits? Is there any shell syntax # that needs more, or should we allow non-ASCII workunit names? self.send_body(bytes(wu_text, "ascii")) except socket_error as e: self.log_error("Connection error: %s", str(e))
def do_upload(dburi, uploaddir, inputfp=sys.stdin, output=sys.stdout, environ=os.environ): diag(1, "Command line arguments:", sys.argv) diag(2, "Environment:", os.environ) try: # Windows needs stdio set for binary mode. # pylint: disable=F0401 import msvcrt # pylint: disable=E1101 msvcrt.setmode(0, os.O_BINARY) # stdin = 0 msvcrt.setmode(1, os.O_BINARY) # stdout = 1 except ImportError: pass diag(1, "Reading POST data") form = cgi.FieldStorage(fp=inputfp, environ=environ) diag(1, "Finished reading POST data") diag(2, "form = ", form) analyze(3, "form", form) charset = "utf-8" header = "Content-Type: text/plain; charset=" + charset + "\r\n\r\n" # A nested FieldStorage instance holds the file message = None if "WUid" not in form: message = 'No "WUid" key found in POST data' if "clientid" not in form: message = 'No "clientid" key found in POST data' elif not os.path.isdir(uploaddir): message = 'Script error: %s is not a directory' % uploaddir else: wuid = form['WUid'] clientid = form['clientid'] if 'errorcode' in form: errorcode = int(form['errorcode'].value) diag(1, "errorcode = ", errorcode) else: errorcode = None if 'failedcommand' in form: failedcommand = int(form['failedcommand'].value) diag(1, "failedcommand = ", failedcommand) else: failedcommand = None # Test if wuid and clientid was set: if not wuid.value: message = 'No workunit was specified' elif not clientid.value: message = 'No client id was specified' diag(1, "wuid = ", wuid.value) diag(1, "clientid = ", clientid.value) if not message: filetuples = [] if 'results' in form: fileitems = form['results'] if isinstance(fileitems, cgi.FieldStorage): fileitems = [fileitems] # Make it iterable else: fileitems = [] diag(1, 'No "results" form found') analyze(3, "fileitems", fileitems) message = "" for fileitem in fileitems: if not fileitem.file: continue analyze(3, "f", fileitem) diag(1, "Processing file ", fileitem.filename) # strip leading path from file name to avoid directory traversal # attacks basename = os.path.basename(fileitem.filename) # Split extension from file name. We need to preserve the # file extension so that, e.g., gzipped files can be identified # as such (basename, suffix) = os.path.splitext(basename) # Make a file name which does not exist yet and create the file (filedesc, filename) = mkstemp(prefix=basename + '.', suffix=suffix, dir=uploaddir) diag(1, "output filename = ", filename) filestuple = [fileitem.filename, filename] # mkstmp() creates files with mode 0o600 (before umask), and does # not allow overriding this with a parameter. We change the mode # to 666 & ~umask. if os.name != "nt": # The os.umask() function gets the old umask *and* sets a new # one, so we have to call it twice to avoid changing it :( umask = os.umask(0o022) os.umask(umask) filemode = 0o666 & ~umask diag(1, "Setting %s to mode %o" % (filename, filemode)) os.fchmod(filedesc, filemode) filetype = fileitem.headers.get("filetype", None) if not filetype is None: filestuple.append(filetype) diag(1, "filetype = ", filetype) command = fileitem.headers.get("command", None) if not command is None: filestuple.append(command) diag(1, "command = ", command) if False: filestuple[1] = os.path.basename(filestuple[1]) filetuples.append(filestuple) # fd is a file descriptor, make a file object from it diag(1, "Getting file object for temp file") file = os.fdopen(filedesc, "wb") diag(1, "Writing data to temp file") try: copyfileobj(fileitem.file, file) except OSError as e: if e.errno == errno.ENOSPC: sys.stderr.write("Disk full, deleting %s\n" % filename) file.close() os.unlink(filename) raise nr_bytes = file.tell() diag(1, "Wrote %d bytes" % nr_bytes) diag(1, "Closing file") file.close() # Example output: # upload.py: The file "testrun.polyselect.0-5000" for workunit # testrun_polyselect_0-5000 was uploaded successfully by client # localhost and stored as /localdisk/kruppaal/work/testrun.upload/ # testrun.polyselect.0-5000.kcudj7, received 84720 bytes. message += 'The file "%s" for workunit %s was uploaded ' \ 'successfully by client %s and stored as %s, received %d bytes.\n' \ % (basename, wuid.value, clientid.value, filename, nr_bytes) if errorcode: message += 'Error code = %d.\n' % errorcode diag(1, "Getting WuAccess object") wuar = wudb.WuAccess(dburi) diag(1, "Got WuAccess object. Calling .result()") try: wuar.result(wuid.value, clientid.value, filetuples, errorcode, failedcommand) except wudb.StatusUpdateError: message = 'Workunit ' + wuid.value + 'was not currently assigned' else: message = message + 'Workunit ' + wuid.value + ' completed.\n' diag(1, "Finished .result()") diag(1, sys.argv[0] + ': ', message.rstrip("\n")) if output == sys.stdout: output.write(header + message) else: output.write((header + message).encode(charset))
'--received', action='store_true', help='Process all workunits currently marked as received without error' ) parser.add_argument( '--update', action='store_true', help= 'Update status of processed WUs to verified with/without error according to test result' ) args = parser.parse_args() if "wudb" in args: wudb_name = args.wudb acc = wudb.WuAccess(wudb_name) if "file" in args: # find wu for this file # do_check(acc, wu) pass if args.received: # Test all workunits whose status is RECEIVED_OK received = acc.query(eq={"status": wudb.WuStatus.RECEIVED_OK}) badidealsfile = None print("Found " + str(len(received)) + " workunits") for wu in received: result = do_check(acc, wu, list(args.program)) if result is None: print(