def send_query(self):
splitpath = urllib.parse.urlsplit(self.path)
conditions = {}
logging.debug("send_query(): query: %s", splitpath.query)
if splitpath.query:
# Parse query part into SELECT conditions
# Now look at individual key=value pairs
for query in splitpath.query.split("&"):
query = urllib.parse.unquote_plus(query)
# logging.debug("Processing token " + str(query))
for (name, op) in wudb.MyCursor.name_to_operator.items():
if op in query:
(key, value) = query.split(op, 1)
key = key.strip()
value = value.strip()
if not name in conditions:
conditions[name] = {}
# If value is of the form "now(-123)", convert it to a
# time stamp of 123 minutes ago
r = re.match(r"now\((-?\d+)\)", value)
if r:
minutes_ago = int(r.group(1))
td = datetime.timedelta(minutes=minutes_ago)
value = str(datetime.datetime.utcnow() + td)
conditions[name][key] = value
break
logging.debug("send_query(): conditions: %s", conditions)
if self.db_pool:
wus = self.db_pool.query(**conditions)
else:
wus = wudb.WuAccess(self.dbdata).query(**conditions)
body = HtmlGen()
body.append('<a href="/index.html">Back to index</a>')
body.append("<p>Query for conditions = " + str(conditions) + "</p>")
if not wus is None and len(wus) > 0:
cwd = os.getcwd()
body.append(str(len(wus)) + " records match.")
keys = wus[0].keys()
body.start_table(keys)
for wu in wus:
body.wu_row(wu, keys, cwd)
body.end_table()
else:
body.append("No records match.")
body.finish()
self.send_response(200)
self.send_header("Content-Type", "text/html")
self.send_header("Cache-Control", "no-cache")
self.send_header("Content-Length", body.get_len())
self.end_headers()
self.send_body(body.__bytes__())
def send_WU(self):
splitpath = urllib.parse.urlsplit(self.path)
# print(str(splitpath))
clientid_match = self.clientid_pattern.match(splitpath.query)
if not clientid_match:
return self.send_error(400, "No or malformed client id specified")
clientid = clientid_match.group(1)
if not self.serving_wus[0]:
try:
return self.send_error(410, "Distributed computation finished")
except socket_error as e:
self.log_error("Connection error: %s", str(e))
return
if self.db_pool:
wu_text = self.db_pool.assign(clientid,
timeout_hint=self.timeout_hint)
else:
wu_text = wudb.WuAccess(self.dbdata).assign(
clientid, timeout_hint=self.timeout_hint)
if not wu_text:
# This flag is to downgrade the logging level. Ugly.
self.no_work_available = True
try:
return self.send_error(404, "No work available")
except socket_error as e:
self.log_error("Connection error: %s", str(e))
return
self.log_message("Sending workunit " + Workunit(wu_text).get_id() +
" to client " + clientid)
# wu_text = wu.get_wu()
try:
self.send_response(200)
self.send_header("Content-Type", "text/plain")
self.send_header("Content-Length", len(wu_text))
self.send_header("Cache-Control", "no-cache")
self.end_headers()
# FIXME: is ASCII enough for workunits? Is there any shell syntax
# that needs more, or should we allow non-ASCII workunit names?
self.send_body(bytes(wu_text, "ascii"))
except socket_error as e:
self.log_error("Connection error: %s", str(e))
def do_upload(dburi,
uploaddir,
inputfp=sys.stdin,
output=sys.stdout,
environ=os.environ):
diag(1, "Command line arguments:", sys.argv)
diag(2, "Environment:", os.environ)
try: # Windows needs stdio set for binary mode.
# pylint: disable=F0401
import msvcrt
# pylint: disable=E1101
msvcrt.setmode(0, os.O_BINARY) # stdin = 0
msvcrt.setmode(1, os.O_BINARY) # stdout = 1
except ImportError:
pass
diag(1, "Reading POST data")
form = cgi.FieldStorage(fp=inputfp, environ=environ)
diag(1, "Finished reading POST data")
diag(2, "form = ", form)
analyze(3, "form", form)
charset = "utf-8"
header = "Content-Type: text/plain; charset=" + charset + "\r\n\r\n"
# A nested FieldStorage instance holds the file
message = None
if "WUid" not in form:
message = 'No "WUid" key found in POST data'
if "clientid" not in form:
message = 'No "clientid" key found in POST data'
elif not os.path.isdir(uploaddir):
message = 'Script error: %s is not a directory' % uploaddir
else:
wuid = form['WUid']
clientid = form['clientid']
if 'errorcode' in form:
errorcode = int(form['errorcode'].value)
diag(1, "errorcode = ", errorcode)
else:
errorcode = None
if 'failedcommand' in form:
failedcommand = int(form['failedcommand'].value)
diag(1, "failedcommand = ", failedcommand)
else:
failedcommand = None
# Test if wuid and clientid was set:
if not wuid.value:
message = 'No workunit was specified'
elif not clientid.value:
message = 'No client id was specified'
diag(1, "wuid = ", wuid.value)
diag(1, "clientid = ", clientid.value)
if not message:
filetuples = []
if 'results' in form:
fileitems = form['results']
if isinstance(fileitems, cgi.FieldStorage):
fileitems = [fileitems] # Make it iterable
else:
fileitems = []
diag(1, 'No "results" form found')
analyze(3, "fileitems", fileitems)
message = ""
for fileitem in fileitems:
if not fileitem.file:
continue
analyze(3, "f", fileitem)
diag(1, "Processing file ", fileitem.filename)
# strip leading path from file name to avoid directory traversal
# attacks
basename = os.path.basename(fileitem.filename)
# Split extension from file name. We need to preserve the
# file extension so that, e.g., gzipped files can be identified
# as such
(basename, suffix) = os.path.splitext(basename)
# Make a file name which does not exist yet and create the file
(filedesc, filename) = mkstemp(prefix=basename + '.',
suffix=suffix,
dir=uploaddir)
diag(1, "output filename = ", filename)
filestuple = [fileitem.filename, filename]
# mkstmp() creates files with mode 0o600 (before umask), and does
# not allow overriding this with a parameter. We change the mode
# to 666 & ~umask.
if os.name != "nt":
# The os.umask() function gets the old umask *and* sets a new
# one, so we have to call it twice to avoid changing it :(
umask = os.umask(0o022)
os.umask(umask)
filemode = 0o666 & ~umask
diag(1, "Setting %s to mode %o" % (filename, filemode))
os.fchmod(filedesc, filemode)
filetype = fileitem.headers.get("filetype", None)
if not filetype is None:
filestuple.append(filetype)
diag(1, "filetype = ", filetype)
command = fileitem.headers.get("command", None)
if not command is None:
filestuple.append(command)
diag(1, "command = ", command)
if False:
filestuple[1] = os.path.basename(filestuple[1])
filetuples.append(filestuple)
# fd is a file descriptor, make a file object from it
diag(1, "Getting file object for temp file")
file = os.fdopen(filedesc, "wb")
diag(1, "Writing data to temp file")
try:
copyfileobj(fileitem.file, file)
except OSError as e:
if e.errno == errno.ENOSPC:
sys.stderr.write("Disk full, deleting %s\n" % filename)
file.close()
os.unlink(filename)
raise
nr_bytes = file.tell()
diag(1, "Wrote %d bytes" % nr_bytes)
diag(1, "Closing file")
file.close()
# Example output:
# upload.py: The file "testrun.polyselect.0-5000" for workunit
# testrun_polyselect_0-5000 was uploaded successfully by client
# localhost and stored as /localdisk/kruppaal/work/testrun.upload/
# testrun.polyselect.0-5000.kcudj7, received 84720 bytes.
message += 'The file "%s" for workunit %s was uploaded ' \
'successfully by client %s and stored as %s, received %d bytes.\n' \
% (basename, wuid.value, clientid.value, filename, nr_bytes)
if errorcode:
message += 'Error code = %d.\n' % errorcode
diag(1, "Getting WuAccess object")
wuar = wudb.WuAccess(dburi)
diag(1, "Got WuAccess object. Calling .result()")
try:
wuar.result(wuid.value, clientid.value, filetuples, errorcode,
failedcommand)
except wudb.StatusUpdateError:
message = 'Workunit ' + wuid.value + 'was not currently assigned'
else:
message = message + 'Workunit ' + wuid.value + ' completed.\n'
diag(1, "Finished .result()")
diag(1, sys.argv[0] + ': ', message.rstrip("\n"))
if output == sys.stdout:
output.write(header + message)
else:
output.write((header + message).encode(charset))
'--received',
action='store_true',
help='Process all workunits currently marked as received without error'
)
parser.add_argument(
'--update',
action='store_true',
help=
'Update status of processed WUs to verified with/without error according to test result'
)
args = parser.parse_args()
if "wudb" in args:
wudb_name = args.wudb
acc = wudb.WuAccess(wudb_name)
if "file" in args:
# find wu for this file
# do_check(acc, wu)
pass
if args.received:
# Test all workunits whose status is RECEIVED_OK
received = acc.query(eq={"status": wudb.WuStatus.RECEIVED_OK})
badidealsfile = None
print("Found " + str(len(received)) + " workunits")
for wu in received:
result = do_check(acc, wu, list(args.program))
if result is None:
print(