def _encode(string, encoding):
if sys.version_info[0] > 2:
return string.encode(encoding=encoding, errors='strict')
else:
if type(u('')) == type(string):
string = string.encode(encoding)
return string
def log(self, header, string):
f = open(self._log, 'a+')
f.write(header.encode( "utf-8" ) + string.encode( "utf-8" ))
print header.encode( "utf-8" ) + string.encode( "utf-8" )
print "\n"
f.write("\n")
f.close()
def encode(self, string):
clean_sentence_unwantedchars= '["\t\n ]+'
string = string.encode('utf8')
string = string.decode('utf-8')
string = re.sub(clean_sentence_unwantedchars, ' ', string)
string = string.encode('ascii', 'replace').encode('utf-8')
string = string.decode('utf-8')
return str(string)
def only_iso88591(self, string):
flag = True
try:
string.encode("iso-8859-1")
except UnicodeEncodeError:
flag = False
return flag
def getHtml(self, url):
try:
request_score = urllib2.Request(url, headers=self.headers)
response_score = self.opener.open(request_score)
print('claw page')
return response_score.read().decode("gb2312", 'ignore').encode("utf8")
except urllib2.URLError, e:
if hasattr(e, "reason"):
string = "连接bbs 失败, 原因" + str(e.reason)
print string.encode(self.charaterset)
return None
def hash_string(string):
"""
Convenience wrapper that returns the hash of a string.
"""
assert isinstance(string, str), f'{string} is not a string!'
string = string.encode('utf-8')
return sha256(string).hexdigest()
def _canonical_string_encoder(string):
"""
<Purpose>
Encode 'string' to canonical string format.
<Arguments>
string:
The string to encode.
<Exceptions>
None.
<Side Effects>
None.
<Returns>
A string with the canonical-encoded 'string' embedded.
"""
string = '"%s"' % re.sub(r'(["\\])', r'\\\1', string)
if isinstance(string, unicode):
return string.encode('utf-8')
else:
return string
def update(self, view):
if not self.need_upd:
return
self.need_upd = False
color_scheme_path = self.color_scheme_path(view)
if not color_scheme_path:
return
packages_path, cs = color_scheme_path
cont = self.get_color_scheme(packages_path, cs)
current_colors = set("#%s" % c for c in re.findall(r'<string>%s(.*?)</string>' % self.prefix, cont, re.DOTALL))
string = ""
for col, name in self.colors.items():
if col not in current_colors:
fg_col = self.get_fg_col(col)
string += self.gen_string % (self.name, name, col, fg_col, fg_col)
if string:
# edit cont
n = cont.find("<array>") + len("<array>")
try:
cont = cont[:n] + string + cont[n:]
except UnicodeDecodeError:
cont = cont[:n] + string.encode("utf-8") + cont[n:]
self.write_file(packages_path, cs, cont)
self.need_restore = True
def send_raw(self, string):
"""Send raw string to the server.
The string will be padded with appropriate CR LF.
"""
# The string should not contain any carriage return other than the
# one added here.
if '\n' in string:
raise InvalidCharacters(
"Carriage returns not allowed in privmsg(text)")
bytes = string.encode('utf-8') + b'\r\n'
# According to the RFC http://tools.ietf.org/html/rfc2812#page-6,
# clients should not transmit more than 512 bytes.
if len(bytes) > 512:
raise MessageTooLong(
"Messages limited to 512 bytes including CR/LF")
if self.socket is None:
raise ServerNotConnectedError("Not connected.")
sender = getattr(self.socket, 'write', self.socket.send)
try:
sender(bytes)
log.debug("TO SERVER: %s", string)
except socket.error:
# Ouch!
self.disconnect("Connection reset by peer.")
def _header(self, r):
""" Build the contents of the X-NFSN-Authentication HTTP header. See
https://members.nearlyfreespeech.net/wiki/API/Introduction for
more explanation. """
login = self.login
timestamp = self._timestamp()
salt = self._salt()
api_key = self.api_key
request_uri = urlparse(r.url).path
body = ''.encode('utf-8')
if r.body:
body = r.body.encode('utf-8')
body_hash = hashlib.sha1(body).hexdigest()
log.debug("login: %s", login)
log.debug("timestamp: %s", timestamp)
log.debug("salt: %s", salt)
log.debug("api_key: %s", api_key)
log.debug("request_uri: %s", request_uri)
log.debug("body_hash: %s", body_hash)
string = ';'.join((login, timestamp, salt, api_key, request_uri, body_hash))
log.debug("string to be hashed: %s", string)
string_hash = hashlib.sha1(string.encode('utf-8')).hexdigest()
log.debug("string_hash: %s", string_hash)
return ';'.join((login, timestamp, salt, string_hash))
def normalize(string):
string = string.replace(u"Ä", "Ae").replace(u"ä", "ae")
string = string.replace(u"Ö", "Oe").replace(u"ö", "oe")
string = string.replace(u"Ü", "Ue").replace(u"ü", "ue")
string = string.replace(u"ß", "ss")
string = string.encode("ascii", "ignore")
return string
def create(cls, app, redirect_uri = None):
now = datetime.utcnow().replace(tzinfo=utc)
code_expires = now + timedelta(minutes = 10)
string = app.client_id
code = hashlib.sha224(string.encode('utf-8') + now.strftime(settings.DATE_FORMAT).encode('utf-8')).hexdigest()
token = cls(app_id=app, code=code, code_expires=code_expires, redirect_uri=redirect_uri)
return token
def aimlize(self, string):
self.steps = []
string = string.encode('utf-8')
for(module) in self.modules:
string = module.process(string)
self.steps.append(module.getModuleName()+": "+string)
return string
def sign(self):
string = '&'.join(['%s=%s' % (key.lower(), self.ret[key]) for key in sorted(self.ret)])
logger.debug('string:%s' % (string))
signature = hashlib.sha1(string.encode('utf8')).hexdigest()
self.ret['signature'] = signature
logger.debug('signature:%s' % (signature))
return self.ret
def default_filter_hex(pattern, value):
if value == 0:
return False
string = struct.pack(pattern.packchar, value)
hexstr = string.encode('hex')
if hexstr.count('00') > pattern.size/2:
return False
return True
def make_safe_digest(string: str,
hash_func: Callable[[bytes], Any]=hashlib.sha1) -> str:
"""
return a hex digest of `string`.
"""
# hashlib.sha1, md5, etc. expect bytes, so non-ASCII strings must
# be encoded.
return hash_func(string.encode('utf-8')).hexdigest()
def unquote_plus(string, encoding='utf-8', errors='strict'):
'''``urllib.parse.unquote_plus`` with Python 2 compatbility.'''
if sys.version_info[0] == 2:
return urllib.parse.unquote_plus(
string.encode(encoding, errors)
).decode(encoding, errors)
else:
return urllib.parse.unquote_plus(string, encoding, errors)
def send_raw(self, string):
"""Send raw string to the server.
The string will be padded with appropriate CR LF.
"""
if self.socket is None:
raise ServerNotConnectedError, "Not connected."
try:
if self.ssl:
self.ssl.write(string.encode('utf-8', 'ignore') + "\r\n")
else:
self.socket.send(string.encode('utf-8', 'ignore') + "\r\n")
if DEBUG:
print "TO SERVER:", string
except socket.error, x:
# Ouch!
self.disconnect("Connection reset by peer.")
def _normalize(self, string):
# convert strings like 'ogd_datatype:datenaggregat' to 'Datenaggregat'
match = re.search(r'^ogd_.*:(.*)$', string)
if match:
string = match.group(1).capitalize()
if type(string) == unicode:
return string.encode('utf8', 'ignore')
else:
return str(string)
def mb_code(string, coding="utf-8"):
if isinstance(string, unicode):
return string.encode(coding)
for c in ('utf-8', 'gb2312', 'gbk', 'gb18030', 'big5'):
try:
return string.decode(c).encode(coding)
except:
pass
return string
def scrubstring(string):
from scrubber import Scrubber
scrubber = Scrubber(autolink=True)
try:
string = string.decode('ascii')
except UnicodeDecodeError:
string = string.decode('utf-8')
string = scrubber.scrub(string)
return string.encode('utf-8')
def sign(self):
ticket = self.get_ticket()
self.ret['jsapi_ticket'] = ticket
string = '&'.join(['%s=%s' % (key.lower(), self.ret[key]) for key in sorted(self.ret)])
logger.debug('string:%s' % (string))
signature = hashlib.sha1(string.encode('utf8')).hexdigest()
self.ret['signature'] = signature
logger.debug('signature:%s' % (signature))
return self.make_wechat_config()
def quote_plus(string, safe='', encoding='utf-8', errors='strict'):
'''``urllib.parse.quote_plus`` with Python 2 compatbility.'''
if sys.version_info[0] == 2:
# Backported behavior
return urllib.parse.quote_plus(
string.encode(encoding, errors),
safe.encode(encoding, errors)
).decode(encoding, errors)
else:
return urllib.parse.quote_plus(string, safe, encoding, errors)
def fsencode(string):
u"""Encode byte strings from unicode with file system encoding.
This function is modelled after its namesake in the Python 3 os.path module.
"""
if isinstance(string, unicode):
return string.encode( sys.getfilesystemencoding() )
elif isinstance(string, str):
return string
else:
raise TypeError("argument is not of string type: {!r}".format(string))
def Print(self,string):
''' Printing method '''
if isinstance(string, unicode):
string = string.encode('utf8')
if self.inTerminal:
print "%s" % string
else:
self.File.write(string+'\n')
self.File.flush()
def detectionPeoples(self, image, compress = False):
function = "detectPeoples"
string = image.tostring()
if compress:
function = "detectPeoplesCompress"
string = string.encode( "zlib" )
#compress = image.tostring().encode("zlib")
base = base64.b64encode( string )
#print "Délka dat:", len(base64.b64encode(image.tostring()) ), len(base)
param = [ str(image.shape[0]), str(image.shape[1]), base ]
r = self.sendMessageWaiting( function, param, "returnDetectPeoples" )
return self.returnDetectionPeoples( *r[1:] )
def toValidString(string):
if string is None or (len(string) < 1):
return 'xx'.encode('utf-8')
assert(isinstance(string, StringTypes))
temp = string.encode('utf-8')
temp = temp.strip() # remove leading and trailing whitespace characters
if len(temp) < 1:
return '-'.encode('utf-8') ;
return extractAlphanumeric(temp)
def check_utf8(string):
"""
Validate if a string is valid UTF-8 str or unicode and that it
does not contain any null character.
:param string: string to be validated
:returns: True if the string is valid utf-8 str or unicode and
contains no null characters, False otherwise
"""
if not string:
return False
try:
if isinstance(string, unicode):
string.encode('utf-8')
else:
string.decode('UTF-8')
return '\x00' not in string
# If string is unicode, decode() will raise UnicodeEncodeError
# So, we should catch both UnicodeDecodeError & UnicodeEncodeError
except UnicodeError:
return False
def privmsg(self, string, encoding=""):
"""Send data to DCC peer.
The string will be padded with appropriate LF if it's a DCC
CHAT session.
"""
try:
if encoding == "":
self.socket.send(string.encode(self.textencoding))
else:
self.socket.send(string.encode(encoding))
if self.dcctype == "chat":
if encoding == "":
self.socket.send("\n".encode(self.textencoding))
else:
self.socket.send("\n".encode(encoding))
if DEBUG:
print("TO PEER: {0}\n".format(string))
except socket.error as x:
# Ouch!
self.disconnect("Connection reset by peer.")
def send_raw_command(self, string):
# Python 3.3 string literals are stupid
#utf = bytes((string + "\r\n"), 'utf-8')
# Printing Unicode characters on Windows = exception
try:
print("Send: " + string)
except:
print("Send: Unicode error(?)")
string += "\r\n"
self.sock.send(string.encode("utf-8"))
def test_put_key(self, etcd, string):
etcd.put('/doot/put_1', string)
out = etcdctl('get', '/doot/put_1')
assert base64.b64decode(out['kvs'][0]['value']) == \
string.encode('utf-8')
def generate_vote_hash(quote, user):
string = str(quote.pk) + user + VoteKey.get()
return hashlib.md5(string.encode("utf-8")).hexdigest()
def hashstring(string, algo):
return hashlib.new(algo, string.encode('utf-8')).hexdigest()
def uni(string):
if isinstance(string, basestring):
if isinstance(string, unicode):
string = string.encode('utf-8', 'ignore')
return string
def data_append_string(self, string):
string = bytes(string.encode(self.encoding) + b'\0')
self.data_append_auto(string)
def strip_lone_surrogates(string):
"""Removes lone surrogates."""
return string.encode("utf-8", "surrogatepass").decode("utf-8", "ignore")
def lfs_quote(string, encoding="utf-8"):
"""Encodes passed string to passed encoding before quoting with
urllib.quote().
"""
return urllib.quote(string.encode(encoding))
def sha256(string):
# print("sha256()")
global sha256c
sha256c = sha256c + 1
return hashlib.sha256(string.encode('UTF-8')).hexdigest()
def sha512(string):
# print("sha512()")
sharandom.sha512c = sharandom.sha512c + 1
return hashlib.sha512(string.encode('UTF-8')).hexdigest()
def normalize(cls, string):
if isinstance(string, str):
string = string.encode()
string = string.translate(_normalize_crockford_b32)
return string.decode()
def md5(string):
h = hashlib.md5()
h.update(string.encode())
return h.hexdigest()
def sign(self):
string = '&'.join(['%s=%s' % (key.lower(), self.ret[key]) for key in sorted(self.ret)])
print(string)
self.ret['signature'] = hashlib.sha1(string.encode('utf-8')).hexdigest()
return self.ret
def executeTestcases(complete_config_obj, tmp_logs, tmp_work, queue_obj,
submission_string, item_name, USE_DOCKER,
OBSOLETE_CONTAINER, which_untrusted, job_id,
grading_began):
queue_time_longstring = queue_obj["queue_time"]
waittime = queue_obj["waittime"]
is_batch_job = queue_obj["regrade"]
job_id = queue_obj["job_id"]
is_batch_job_string = "BATCH" if is_batch_job else "INTERACTIVE"
runner_success = -1
first_testcase = True
# run the run.out as the untrusted user
with open(os.path.join(tmp_logs, "runner_log.txt"), 'w') as logfile:
print("LOGGING BEGIN my_runner.out", file=logfile)
logfile.flush()
testcases = complete_config_obj["testcases"]
# we start counting from one.
for testcase_num in range(1, len(testcases) + 1):
if 'type' in testcases[testcase_num - 1]:
if testcases[testcase_num -
1]['type'] == 'FileCheck' or testcases[
testcase_num - 1]['type'] == 'Compilation':
continue
#make the tmp folder for this testcase.
testcase_folder = os.path.join(tmp_work,
"test{:02}".format(testcase_num))
os.makedirs(testcase_folder)
os.chdir(testcase_folder)
if USE_DOCKER:
try:
use_router = testcases[testcase_num - 1]['use_router']
single_port_per_container = testcases[
testcase_num - 1]['single_port_per_container']
# returns a dictionary where container_name maps to outgoing connections and container image
container_info = find_container_information(
testcases[testcase_num - 1], testcase_num, use_router)
# Creates folders for each docker container if there are more than one. Otherwise, we grade in testcase_folder.
# Updates container_info so that each docker has a 'mounted_directory' element
create_container_subfolders(container_info,
testcase_folder,
which_untrusted)
# Launches containers with the -d option. Gives them the names specified in container_info. Updates container info
# to store container_ids.
launch_containers(container_info, testcase_folder, job_id,
is_batch_job, which_untrusted, item_name,
grading_began, queue_obj,
submission_string, testcase_num)
# Networks containers together if there are more than one of them. Modifies container_info to store 'network'
# The name of the docker network it is connected to.
network_containers(container_info,
os.path.join(tmp_work, "test_input"),
which_untrusted, use_router,
single_port_per_container)
print('NETWORKED CONTAINERS')
#The containers are now ready to execute.
processes = dict()
#Set up the mounted folders before any dockers start running in case large file transfer sizes cause delay.
for name, info in container_info.items():
mounted_directory = info['mounted_directory']
#Copies the code needed to run into mounted_directory
#TODO this can eventually be extended so that only the needed code is copied in.
setup_folder_for_grading(mounted_directory, tmp_work,
job_id, tmp_logs)
# Start the docker containers.
# Start the router first if in router mode
if 'router' in container_info and use_router:
info = container_info['router']
c_id = info['container_id']
mounted_directory = info['mounted_directory']
full_name = '{0}_{1}'.format(which_untrusted, 'router')
print('spinning up docker {0} with c_id {1}'.format(
full_name, c_id))
p = subprocess.Popen(
['docker', 'start', '-i', '--attach', c_id],
stdout=logfile,
stdin=subprocess.PIPE)
processes['router'] = p
time.sleep(1)
for name, info in container_info.items():
if name == 'router' and use_router:
continue
c_id = info['container_id']
mounted_directory = info['mounted_directory']
full_name = '{0}_{1}'.format(which_untrusted, name)
print('spinning up docker {0} with c_id {1}'.format(
full_name, c_id))
p = subprocess.Popen(
['docker', 'start', '-i', '--attach', c_id],
stdout=logfile,
stdin=subprocess.PIPE)
processes[name] = p
# Handle the dispatcher actions
dispatcher_actions = testcases[testcase_num -
1]["dispatcher_actions"]
#if there are dispatcher actions, give the student code a second to start up.
if len(dispatcher_actions) > 0:
time.sleep(1)
#TODO add error handling once we've encountered some errors.
for action_obj in dispatcher_actions:
action_type = action_obj["action"]
if action_type == "delay":
#todo add some protections here.
time_in_seconds = float(action_obj["seconds"])
while time_in_seconds > 0 and at_least_one_alive(
processes):
if time_in_seconds >= .1:
time.sleep(.1)
else:
time.sleep(time_in_seconds)
#can go negative (subtracts .1 even in the else case) but that's fine.
time_in_seconds -= .1
elif action_type == "stdin":
string = action_obj["string"]
targets = action_obj["containers"]
for target in targets:
p = processes[target]
# poll returns None if the process is still running.
if p.poll() == None:
p.stdin.write(string.encode('utf-8'))
p.stdin.flush()
else:
pass
#Now that all dockers are running, wait on their return code for success or failure. If any fail, we count it
# as a total failure.
for name, process in processes.items():
process.wait()
rc = process.returncode
runner_success = rc if first_testcase else max(
runner_success, rc)
first_testcase = False
except Exception as e:
print('An error occurred when grading by docker.')
traceback.print_exc()
finally:
clean_up_containers(container_info, job_id, is_batch_job,
which_untrusted, item_name,
grading_began, use_router)
print("CLEANED UP CONTAINERS")
else:
try:
# Move the files necessary for grading (runner, inputs, etc.) into the testcase folder.
setup_folder_for_grading(testcase_folder, tmp_work, job_id,
tmp_logs)
my_testcase_runner = os.path.join(testcase_folder,
'my_runner.out')
runner_success = subprocess.call([
os.path.join(SUBMITTY_INSTALL_DIR, "sbin",
"untrusted_execute"), which_untrusted,
my_testcase_runner, queue_obj["gradeable"],
queue_obj["who"],
str(queue_obj["version"]), submission_string,
str(testcase_num)
],
stdout=logfile)
except Exception as e:
print("ERROR caught runner.out exception={0}".format(
str(e.args[0])).encode("utf-8"),
file=logfile)
traceback.print_exc()
logfile.flush()
os.chdir(tmp_work)
print("LOGGING END my_runner.out", file=logfile)
logfile.flush()
killall_success = subprocess.call([
os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "untrusted_execute"),
which_untrusted,
os.path.join(SUBMITTY_INSTALL_DIR, "sbin", "killall.py")
],
stdout=logfile)
print("KILLALL COMPLETE my_runner.out", file=logfile)
logfile.flush()
if killall_success != 0:
msg = 'RUNNER ERROR: had to kill {} process(es)'.format(
killall_success)
print("pid", os.getpid(), msg)
grade_items_logging.log_message(job_id, is_batch_job,
which_untrusted, item_name, "", "",
msg)
return runner_success
def multireplace(string, rep):
pattern = re.compile("|".join(rep.keys()))
string = pattern.sub(lambda m: rep[re.escape(m.group(0))],
string.decode('utf-8'))
return string.encode()
def write(string):
temporary_file.write(string.encode('utf-8'))
string = []
string.append(HTTP_method)
string.append('&')
string.append(urllib.parse.quote(url, safe=''))
string.append('&')
string.append(urllib.parse.quote(parameter_str, safe=''))
#string = ''.join(string)
tstring = ''
for item in string:
tstring += item
string = tstring
signing_key = urllib.parse.quote(
'qfGdZ6KcnlVjQHiJKTu3W4QrFxKHi4nSQH5pdjLc78AmfB7VJz', safe='') + '&'
message = string.encode('utf-8')
secret = signing_key.encode('utf-8')
signature = urllib.parse.quote(
base64.standard_b64encode(
hmac.new(signing_key.encode(), string.encode(),
sha1).digest()).decode('ascii'))
print('\n')
print(signature)
DST = []
DST.append('OAuth ')
values = {
'oauth_nonce': oauth_nonce,
'oauth_callback': oauth_callback,
def _latinize(self, string):
'''
Replaces unicode characters with closest Latin equivalent. For example,
Alejandro González Iñárritu becomes Alejando Gonzalez Inarritu.
'''
return unidecode(string.encode().decode('utf-8'))
def utf8(string):
if isinstance(string, str):
return string.encode("utf-8")
return string
def _stringify(string):
# convert to 7-bit ascii if possible
try:
return string.encode("ascii")
except UnicodeError:
return string
def to_string(string):
"""Converts unicode to utf-8 if on Python 2, leaves as is if on Python 3"""
return string.encode('utf-8') if sys.version_info[0] < 3 else string
def asciis(string):
if isinstance(string, basestring):
if isinstance(string, unicode):
string = string.encode('ascii', 'ignore')
return string
def salted_hash(string):
return hashlib.sha1(string.encode("utf-8") + settings.SECRET_KEY.encode("utf-8")).hexdigest()
def sanitize(self, string):
string = string.encode('utf-8')
string = self.replace_all(string, {'\r':'', '\n':''})
return string
def encode_to_binary(self, string):
return string.encode('utf-8')
def test_get_key(self, etcd, string):
etcdctl('put', '/doot/a_key', string)
returned, _ = etcd.get('/doot/a_key')
assert returned == string.encode('utf-8')
def zh_ch(string):
return string.encode("gbk").decode(errors="ignore")
def test_get_key_serializable(self, etcd, key, string):
etcdctl('put', '/doot/' + key, string)
with _out_quorum():
returned, _ = etcd.get('/doot/' + key, serializable=True)
assert returned == string.encode('utf-8')
def _stringify(string):
try:
return string.encode('ascii')
except UnicodeError:
return string
def openOfficeStringUtf8(self, string):
if type(string) == unicode:
return string.encode("utf-8")
tempstring = unicode(string, "cp1252").encode("utf-8")
return tempstring
def encode(string, encoding='utf-8'):
"""If unicode, encode to encoding; if 8-bit string, leave unchanged."""
if isinstance(string, unicode):
string = string.encode(encoding)
return string
