def _get_user_identity(self): """Determine the identity to use for new commits. """ user = os.environ.get("GIT_COMMITTER_NAME") email = os.environ.get("GIT_COMMITTER_EMAIL") config = self.get_config_stack() if user is None: try: user = config.get(("user", ), "name") except KeyError: user = None if email is None: try: email = config.get(("user", ), "email") except KeyError: email = None if user is None: import getpass user = getpass.getuser().encode(sys.getdefaultencoding()) if email is None: import getpass import socket email = ("{}@{}".format(getpass.getuser(), socket.gethostname()) .encode(sys.getdefaultencoding())) return (user + b" <" + email + b">")
def create_pdf(manuscript, journal, authors, publisher, target_agreement): # get a temporary file name pdf_fn = tempfile.NamedTemporaryFile().name # generate the appropriate PDF if target_agreement in agreements.handlers: agreements.handlers[target_agreement]() ( pdf_fn, manuscript=manuscript.decode(sys.getdefaultencoding(),'ignore'), journal=journal.decode(sys.getdefaultencoding(),'ignore'), author=[ a.decode(sys.getdefaultencoding(),'ignore') for a in authors], publisher=publisher.decode(sys.getdefaultencoding(),'ignore')) else: # invalid target show_error() # read the temporary file pdf_file = file(pdf_fn, "rb") result = pdf_file.read() pdf_file.close() # remove the temp file os.unlink(os.path.join(tempfile.tempdir, pdf_fn)) # return the contents return result
def test_astype_unicode(self): # GH7758 # a bit of magic is required to set default encoding encoding to utf-8 digits = string.digits test_series = [ Series([digits * 10, tm.rands(63), tm.rands(64), tm.rands(1000)]), Series([u('データーサイエンス、お前はもう死んでいる')]), ] former_encoding = None if not compat.PY3: # in python we can force the default encoding for this test former_encoding = sys.getdefaultencoding() reload(sys) # noqa sys.setdefaultencoding("utf-8") if sys.getdefaultencoding() == "utf-8": test_series.append(Series([u('野菜食べないとやばい').encode("utf-8")])) for s in test_series: res = s.astype("unicode") expec = s.map(compat.text_type) assert_series_equal(res, expec) # restore the former encoding if former_encoding is not None and former_encoding != "utf-8": reload(sys) # noqa sys.setdefaultencoding(former_encoding)
def ReportFatalException(e, stacktrace_on_error=False): """ Formats various exceptions; raises SystemExit, never returns. """ if isinstance(e, XsltException) or \ isinstance(e, RuntimeException) or \ isinstance(e, CompiletimeException): if stacktrace_on_error: traceback.print_exc(1000, sys.stderr) raise SystemExit(''.join([c.encode(sys.getdefaultencoding(), 'ignore') or "&#%d;" % ord(c) for c in e.message])) else: if stacktrace_on_error: traceback.print_exc(1000, sys.stderr) msg = '' else: exceptiontype = str(sys.exc_type) if exceptiontype.startswith('exceptions.'): exceptiontype = exceptiontype[11:] msg = 'An unexpected error occurred while processing.\n' + \ 'The error was: %s: %s\n' % (exceptiontype, str(e)) + \ 'Use the -e (--stacktrace-on-error) option for a full stack trace.' if msg: raise SystemExit(''.join([c.encode(sys.getdefaultencoding(), 'ignore') or "&#%d;" % ord(c) for c in msg])) else: sys.exit()
def test_unicode(self): """Title can contain unicode characters.""" if 'utf-8' != sys.getdefaultencoding(): raise SkipTest("encoding '%s' can't deal with snowmen" % sys.getdefaultencoding()) rv = self.run_script(r""" snowman = u'\u2603' import setproctitle setproctitle.setproctitle("Hello, " + snowman + "!") import os print os.getpid() print os.popen("ps -o pid,command 2> /dev/null").read() """) lines = filter(None, rv.splitlines()) pid = lines.pop(0) pids = dict([r.strip().split(None, 1) for r in lines]) snowmen = [ u'\u2603', # ps supports unicode r'\M-b\M^X\M^C', # ps output on BSD r'M-bM^XM^C', # ps output on OS-X ] title = self._clean_up_title(pids[pid]) for snowman in snowmen: if title == "Hello, " + snowman + "!": break else: self.fail("unexpected ps output: %r" % title)
def bug_1771_with_user_config(var): # sys.getdefaultencoding is considered as a never # returning function in the inconsistent_returns.rc file. if var == 1: sys.getdefaultencoding() else: return var * 2
def get_git_revision(repopath): """ Return Git revision for the repository located at repopath Result is a tuple (latest commit hash, branch), with None values on error """ try: git = programs.find_program('git') assert git is not None and osp.isdir(osp.join(repopath, '.git')) commit = programs.run_program(git, ['rev-parse', '--short', 'HEAD'], cwd=repopath).communicate() commit = commit[0].strip() if PY3: commit = commit.decode(sys.getdefaultencoding()) # Branch branches = programs.run_program(git, ['branch'], cwd=repopath).communicate() branches = branches[0] if PY3: branches = branches.decode(sys.getdefaultencoding()) branches = branches.split('\n') active_branch = [b for b in branches if b.startswith('*')] if len(active_branch) != 1: branch = None else: branch = active_branch[0].split(None, 1)[1] return commit, branch except (subprocess.CalledProcessError, AssertionError, AttributeError): return None, None
def exec_process(cmdline, silent=True, catch_enoent=True, input=None, **kwargs): """Execute a subprocess and returns the returncode, stdout buffer and stderr buffer. Optionally prints stdout and stderr while running.""" try: sub = subprocess.Popen(args=cmdline, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) stdout, stderr = sub.communicate(input=input) if type(stdout) != type(""): # decode on Python 3 # do nothing on Python 2 (it just doesn't care about encoding anyway) stdout = stdout.decode(sys.getdefaultencoding(), "replace") stderr = stderr.decode(sys.getdefaultencoding(), "replace") returncode = sub.returncode if not silent: sys.stdout.write(stdout) sys.stderr.write(stderr) except OSError as e: if e.errno == errno.ENOENT and catch_enoent: raise DistutilsError('"%s" is not present on this system' % cmdline[0]) else: raise if returncode != 0: raise DistutilsError('Got return value %d while executing "%s", stderr output was:\n%s' % (returncode, " ".join(cmdline), stderr.rstrip("\n"))) return stdout
def test_utf8_html_error_template(self): """test the html_error_template with a Template containing utf8 chars""" if util.py3k: code = """# -*- coding: utf-8 -*- % if 2 == 2: /an error ${'привет'} % endif """ else: code = """# -*- coding: utf-8 -*- % if 2 == 2: /an error ${u'привет'} % endif """ try: template = Template(code) template.render_unicode() except exceptions.CompileException, ce: html_error = exceptions.html_error_template().render() assert ("CompileException: Fragment 'if 2 == 2: /an " "error' is not a partial control " "statement at line: 2 char: 1") in \ html_error.decode('utf-8') if util.py3k: assert u"3 ${'привет'}".encode(sys.getdefaultencoding(), 'htmlentityreplace') in html_error else: assert u"3 ${u'привет'}".encode(sys.getdefaultencoding(), 'htmlentityreplace') in html_error
def detect_console_encoding(): """ Try to find the most capable encoding supported by the console. slightly modified from the way IPython handles the same issue. """ global _initial_defencoding encoding = None try: encoding = sys.stdout.encoding or sys.stdin.encoding except AttributeError: pass # try again for something better if not encoding or 'ascii' in encoding.lower(): try: encoding = locale.getpreferredencoding() except Exception: pass # when all else fails. this will usually be "ascii" if not encoding or 'ascii' in encoding.lower(): encoding = sys.getdefaultencoding() # GH3360, save the reported defencoding at import time # MPL backends may change it. Make available for debugging. if not _initial_defencoding: _initial_defencoding = sys.getdefaultencoding() return encoding
def build_multi_mime_message(content_type_pairs, file_type_pairs): """ Argument: content_type_pairs - A list of tuples [(content, mime-type, filename)] file_type_pairs -- A list of strings formatted as file-path : mime-type """ if len(file_type_pairs) == 0: return "" combined_message = MIMEMultipart() for i in file_type_pairs: (filename, format_type) = i.split(":", 1) filename = filename.strip() format_type = format_type.strip() with open(filename) as filehandle: contents = filehandle.read() sub_message = MIMEText(contents, format_type, sys.getdefaultencoding()) sub_message.add_header('Content-Disposition', 'attachment;' ' filename="%s"' % (filename)) combined_message.attach(sub_message) for i in content_type_pairs: sub_message = MIMEText(i[0], i[1].strip(), sys.getdefaultencoding()) if len(i) <= 3: sub_message.add_header('Content-Disposition', 'attachment; filename="%s"' % (i[2].strip())) else: sub_message.add_header('Content-Disposition', 'attachment;' ' filename="%s"' % ("cs-cloud-init.yaml")) combined_message.attach(sub_message) return str(combined_message)
def encodeASCII(string, language=None): #from Unicodize and plex scanner and other sources if string=="": return "" ranges = [ {"from": ord(u"\u3300"), "to": ord(u"\u33ff")}, {"from": ord(u"\ufe30"), "to": ord(u"\ufe4f")}, {"from": ord(u"\uf900"), "to": ord(u"\ufaff")}, # compatibility ideographs {"from": ord(u"\u30a0"), "to": ord(u"\u30ff")}, {"from": ord(u"\u2e80"), "to": ord(u"\u2eff")}, # Japanese Kana # cjk radicals supplement {"from": ord(u"\u4e00"), "to": ord(u"\u9fff")}, {"from": ord(u"\u3400"), "to": ord(u"\u4dbf")}] # windows: TypeError: ord() expected a character, but string of length 2 found #{"from": ord(u"\U00020000"), "to": ord(u"\U0002a6df")}, #{"from": ord(u"\U0002a700"), "to": ord(u"\U0002b73f")}, #{"from": ord(u"\U0002b740"), "to": ord(u"\U0002b81f")}, #{"from": ord(u"\U0002b820"), "to": ord(u"\U0002ceaf")}, # included as of Unicode 8.0 #{"from": ord(u"\U0002F800"), "to": ord(u"\U0002fa1f")} # compatibility ideographs encodings, encoding = ['iso8859-1', 'utf-16', 'utf-16be', 'utf-8'], ord(string[0]) # if 0 <= encoding < len(encodings): string = string[1:].decode('cp949') if encoding == 0 and language == 'ko' else string[1:].decode(encodings[encoding]) # If we're dealing with a particular language, we might want to try another code page. if sys.getdefaultencoding() not in encodings: try: string = string.decode(sys.getdefaultencoding()) except: pass if not sys.getfilesystemencoding()==sys.getdefaultencoding(): try: string = string.decode(sys.getfilesystemencoding()) except: pass string = string.strip('\0') try: string = unicodedata.normalize('NFKD', string) # Unicode to ascii conversion to corect most characters automatically except: pass try: string = re.sub(RE_UNICODE_CONTROL, '', string) # Strip control characters. except: pass try: string = string.encode('ascii', 'replace') # Encode into Ascii except: pass original_string, string, i = string, list(string), 0 while i < len(string): ### loop through unicode and replace special chars with spaces then map if found ### if ord(string[i])<128: i = i+1 else: #non ascii char char, char2, char3, char_len = 0, "", [], unicodeLen(string[i]) for x in range(0, char_len): char = 256*char + ord(string[i+x]); char2 += string[i+x]; char3.append(string[i+x]) if not x==0: string[i] += string[i+x]; string[i+x]='' try: asian_language = any([mapping["from"] <= ord("".join(char3).decode('utf8')) <= mapping["to"] for mapping in ranges]) except: asian_language = False if char in CHARACTERS_MAP: string[i]=CHARACTERS_MAP.get( char ) elif not asian_language: Log("*Character missing in CHARACTERS_MAP: %d:'%s' , #'%s' %s, string: '%s'" % (char, char2, char2, char3, original_string)) i += char_len return ''.join(string)
def new_dataset(model): """Initializes the dataset.""" # print model.data_path dataset_id = model.id dataset_path = model.data_path print dataset_path for root, dirs, files in os.walk(dataset_path): first = 1 print files for image in files: print image print sys.getdefaultencoding() image = image.encode("utf8") if image.find(".jpg") > 0: print image if image.find("thumb") > 0: continue classname = root.split('/')[-1] print classname if first == 1: new = Class(model.id,classname) db.session.add(new) first = 0 this_class = Class.query.filter(and_(Class.class_name == classname,Class.dataset_id == dataset_id)).first() print dataset_id print image img = Image(dataset_id,this_class.id,os.path.join(root,image),"unlabelled") db.session.add(img) db.session.commit()
def upload2(pyew, doprint=True): """ upload file to virscan.org""" print sys.getdefaultencoding() register_openers() url = r'http://up.virscan.org/up.php' datagen, headers = multipart_encode({"upfile": open(pyew.filename, "rb"), 'UPLOAD_IDENTIFIER' : 'KEYbc7cf6642fc84bf67747f1bbdce597f0', 'langkey' : '1', 'setcookie' : '0', 'tempvar' : '', 'fpath' : 'C:\\fakepath\\'+pyew.filename }) request = urllib2.Request(url, datagen, headers) request.add_header('Host', 'up.virscan.org') request.add_header('Cache-Control', 'max-age=0') request.add_header('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.104 Safari/537.36') request.add_header('Host', 'up.virscan.org') request.add_header('Accept-Encoding', 'gzip, deflate') request.add_header('Accept-Language', 'zh-CN,zh;q=0.8,en;q=0.6') resp = urllib2.urlopen(request).read() if re.findall("innerHTML='(.*?) file upload", resp): print "Upload File Failed" else: print "Upload Success"
def main(): args = sys.argv cur_sys = system() file_name = GLOBALS["db_name"]+"-"+str(options.port)+".log" # if cur_sys == "Darwin": # f = "/Users/"+os.getlogin()+"/Desktop/"+ file_name # elif cur_sys == "Linux": # f = os.getcwd() + "/" + file_name # else: # raise NotImplementedError # args.append("--log_file_prefix=" + f) # logging.basicConfig(filename=f, level=logging.DEBUG) tornado.options.parse_command_line() applicaton = Application() http_server = tornado.httpserver.HTTPServer(applicaton, xheaders=True) http_server.listen(options.port) print("="*50) print("initializing program with port : ", options.port) print("="*50) print("my ip is : ", socket.gethostbyname(socket.gethostname())) print("="*50) print("File system DEFAULT Encoding-type is : ", sys.getdefaultencoding()) print("File system Encoding-type is : ", sys.getfilesystemencoding()) print("="*50) logging.info("File system DEFAULT Encoding-type is : " + str(sys.getdefaultencoding())) logging.info("File system Encoding-type is : " + str(sys.getfilesystemencoding())) ioloop = tornado.ioloop.IOLoop.instance() ioloop.start()
def test_read_write(self, tmpdir): x = tmpdir.join("hello") part = py.builtin._totext("hällo", "utf8") x.write(part) assert x.read() == part x.write(part.encode(sys.getdefaultencoding())) assert x.read() == part.encode(sys.getdefaultencoding())
def test_startLogging(self): """ startLogging() installs FileLogObserver and overrides sys.stdout and sys.stderr. """ origStdout, origStderr = sys.stdout, sys.stderr self._startLoggingCleanup() # When done with test, reset stdout and stderr to current values: fakeFile = StringIO() observer = log.startLogging(fakeFile) self.addCleanup(observer.stop) log.msg("Hello!") self.assertIn("Hello!", fakeFile.getvalue()) self.assertIsInstance(sys.stdout, LoggingFile) self.assertEqual(sys.stdout.level, NewLogLevel.info) encoding = getattr(origStdout, "encoding", None) if not encoding: encoding = sys.getdefaultencoding() self.assertEqual(sys.stdout.encoding.upper(), encoding.upper()) self.assertIsInstance(sys.stderr, LoggingFile) self.assertEqual(sys.stderr.level, NewLogLevel.error) encoding = getattr(origStderr, "encoding", None) if not encoding: encoding = sys.getdefaultencoding() self.assertEqual(sys.stderr.encoding.upper(), encoding.upper())
def view_locale(request): loc_info = "getlocale: " + str(locale.getlocale()) + \ "<br/>getdefaultlocale(): " + str(locale.getdefaultlocale()) + \ "<br/>fs_encoding: " + str(sys.getfilesystemencoding()) + \ "<br/>sys default encoding: " + str(sys.getdefaultencoding()) "<br/>sys default encoding: " + str(sys.getdefaultencoding()) return HttpResponse(loc_info)
def get_build_message(message, outputs_url, logs_urls=None, applied_reviews=None, failed_command=None): """Retrieves build message.""" if logs_urls is None: logs_urls = [] if applied_reviews is None: applied_reviews = [] build_msg = "%s\n\n" % message if applied_reviews: build_msg += "Reviews applied: `%s`\n\n" % applied_reviews if failed_command: build_msg += "Failed command: `%s`\n\n" % failed_command build_msg += ("All the build artifacts available" " at: %s\n\n" % (outputs_url)) logs_msg = '' for url in logs_urls: response = urllib2.urlopen(url) log_content = response.read().decode(sys.getdefaultencoding()) if log_content == '': continue file_name = url.split('/')[-1] logs_msg += "- [%s](%s):\n\n" % (file_name, url) logs_msg += "```\n" log_tail = log_content.split("\n")[-LOG_TAIL_LIMIT:] logs_msg += "\n".join(log_tail) logs_msg += "```\n\n" if logs_msg == '': return build_msg build_msg += "Relevant logs:\n\n%s" % (logs_msg) return build_msg.encode(sys.getdefaultencoding())
def emit(self, record): category = record.name.encode(sys.getdefaultencoding()) msg = record.getMessage() if record.exc_info is not None: msg += "\n" + "".join(traceback.format_exception(*record.exc_info)) msg = msg.encode(sys.getdefaultencoding()) BWLogging.logLevelToBigWorldFunction[record.levelno](category, msg, None)
def execute_shell(cmd, input='', timeout=None): """ :param cmd: :param input: sent to stdin :return: returncode, stdout, stderr. :raise: subprocess.TimeoutExpired (I kill the process!) """ proc_stdin = subprocess.PIPE if input != '' else None proc_input = input if input != '' else None p = subprocess.Popen(cmd, stdin=proc_stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, preexec_fn=os.setsid if timeout else None) # http://stackoverflow.com/a/4791612/801203 try: if proc_input: out, err = p.communicate(bytes(proc_input, encoding=sys.getdefaultencoding()), timeout=timeout) else: out, err = p.communicate(timeout=timeout) except subprocess.TimeoutExpired: logging.debug('Timeout expired, killing the process') os.killpg(os.getpgid(p.pid), signal.SIGTERM) p.communicate() raise return p.returncode,\ str(out, sys.getdefaultencoding()),\ str(err, sys.getdefaultencoding())
def test_startLogging(self): """ startLogging() installs FileLogObserver and overrides sys.stdout and sys.stderr. """ origStdout, origStderr = sys.stdout, sys.stderr self._startLoggingCleanup() # When done with test, reset stdout and stderr to current values: fakeFile = StringIO() observer = log.startLogging(fakeFile) self.addCleanup(observer.stop) log.msg("Hello!") self.assertIn("Hello!", fakeFile.getvalue()) self.assertIsInstance(sys.stdout, log.StdioOnnaStick) self.assertEqual(sys.stdout.isError, False) encoding = getattr(origStdout, "encoding", None) if not encoding: encoding = sys.getdefaultencoding() self.assertEqual(sys.stdout.encoding, encoding) self.assertIsInstance(sys.stderr, log.StdioOnnaStick) self.assertEqual(sys.stderr.isError, True) encoding = getattr(origStderr, "encoding", None) if not encoding: encoding = sys.getdefaultencoding() self.assertEqual(sys.stderr.encoding, encoding)
def send_email(self, to_, from_=None, subject=None, body=None, subtype="plain", charset="utf-8"): message = MIMEText(body, subtype, charset) if subject: subject_header = Header() subject = (codecs.decode(bytearray(subject, sys.getdefaultencoding()), charset) if isinstance(subject, str) else subject) subject_header.append(subject.strip()) message["Subject"] = subject_header from_ = from_ or self.default_sender from_ = (codecs.decode(bytearray(from_, sys.getdefaultencoding()), charset) if isinstance(from_, str) else from_) from_realname, from_addr = parseaddr(from_) from_header = Header() from_header.append(formataddr((from_realname, from_addr))) message['From'] = from_header to_ = (codecs.decode(bytearray(to_, sys.getdefaultencoding()), charset) if isinstance(to_, str) else to_) to_realname, to_addr = parseaddr(to_) to_header = Header() to_header.append(formataddr((to_realname, to_addr))) message['To'] = to_header self._send(message, from_addr, to_addr)
def send_html_email(self, to_, from_=None, subject=None, text=None, html=None, charset="utf-8"): message = MIMEMultipart("alternative") if subject: subject_header = Header() subject = (codecs.decode(bytearray(subject, sys.getdefaultencoding()), charset) if isinstance(subject, str) else subject) subject_header.append(subject.strip()) message["Subject"] = subject_header from_ = from_ or self.default_sender from_ = (codecs.decode(bytearray(from_, sys.getdefaultencoding()), charset) if isinstance(from_, str) else from_) from_realname, from_addr = parseaddr(from_) from_header = Header() from_header.append(formataddr((from_realname, from_addr))) message['From'] = from_header to_ = (codecs.decode(bytearray(to_, sys.getdefaultencoding()), charset) if isinstance(to_, str) else to_) to_realname, to_addr = parseaddr(to_) to_header = Header() to_header.append(formataddr((to_realname, to_addr))) message['To'] = to_header message.attach(MIMEText(text, "plain", charset)) message.attach(MIMEText(html, "html", charset)) self._send(message, from_addr, to_addr)
def unzipDataFile(self, dataFileName, unzipFileName): print sys.getdefaultencoding() file_object_Z = open(dataFileName, "rb") file_object_W = open(unzipFileName, "wb") try: nDictCnt = 0 while True: urlLen = struct.unpack("L",file_object_Z.read(4)) srcUrl = file_object_Z.read(urlLen[0]) htmlLen = struct.unpack("L",file_object_Z.read(4)) zipHtml = file_object_Z.read(htmlLen[0]) srcHtml = zlib.decompress(zipHtml) htmlLen = struct.pack("L",len(srcHtml)) urlLen = struct.pack("L",urlLen[0]) file_object_W.write(urlLen) file_object_W.write(srcUrl) file_object_W.write(htmlLen) file_object_W.write(srcHtml) #print srcHtml #break except IOError, e: print "IOError:", e
def body(self): r''' Extract the plain text body of the message with signatures stripped off. :param message: the message to extract the body from :type message: :class:`notmuch.Message` :returns: the extracted text body :rtype: :class:`list` of :class:`str`o ''' if self._body is None: content = [] for part in self.mail.walk(): if part.get_content_type() == 'text/plain': raw_payload = part.get_payload(decode=True) encoding = part.get_content_charset() if encoding: try: raw_payload = raw_payload.decode(encoding, 'replace') except LookupError: raw_payload = raw_payload.decode( sys.getdefaultencoding(), 'replace') else: raw_payload = raw_payload.decode( sys.getdefaultencoding(), 'replace') lines = raw_payload.split('\n') lines = self._strip_signatures(lines) content.append('\n'.join(lines)) self._body = '\n'.join(content) return self._body
def runCommand(cmd, instream = None, msg = '', upon_succ = None, show_stderr = False, return_zero = True): if isinstance(cmd, str): cmd = shlex.split(cmd) popen_env = os.environ.copy() popen_env.update(env.path) try: tc = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE, env=popen_env) if instream: instream = instream.encode(sys.getdefaultencoding()) out, error = tc.communicate(instream) else: out, error = tc.communicate() out = out.decode(sys.getdefaultencoding()) error = error.decode(sys.getdefaultencoding()) if return_zero: if tc.returncode < 0: raise ValueError ("Command '{0}' was terminated by signal {1}".format(cmd, -tc.returncode)) elif tc.returncode > 0: raise ValueError ("{0}".format(error)) if error.strip() and show_stderr: env.logger.error(error) except OSError as e: raise OSError ("Execution of command '{0}' failed: {1}".format(cmd, e)) # everything is OK if upon_succ: # call the function (upon_succ) using others as parameters. upon_succ[0](*(upon_succ[1:])) return out.strip(), error.strip()
def DetectEncodingAndRead(self, fd): encodings = ["utf-8", "utf-16"] if locale.getpreferredencoding() not in encodings: encodings.append(locale.getpreferredencoding()) if sys.getdefaultencoding() not in encodings: encodings.append(sys.getdefaultencoding()) if locale.getdefaultlocale()[1] not in encodings: encodings.append(locale.getdefaultlocale()[1]) if sys.getfilesystemencoding() not in encodings: encodings.append(sys.getfilesystemencoding()) if 'latin-1' not in encodings: encodings.append('latin-1') for enc in encodings: fd.seek(0) try: reader = codecs.getreader(enc)(fd) content = reader.read() except: continue else: self._encoding = enc logger.info("Detect file %s 's encoding is %s" % (self.GetFilename(), self._encoding)) return content logger.error("Fail to detect the encoding for file %s" % self.GetFilename()) return None
def main(query, exclude): print sys.getdefaultencoding() createWorkingDirectories(query) getGooogleResults(query, exclude) file = open("researchResults/" + query + "/data/searchResults.txt") line = file.readline() while line:
def __init__(self, msg, stderr=None, stdout=None): # type: (str, bytes, bytes) -> None if stderr: msg += '\n[stderr]\n' + stderr.decode(sys.getdefaultencoding(), 'replace') if stdout: msg += '\n[stdout]\n' + stdout.decode(sys.getdefaultencoding(), 'replace') super().__init__(msg)
from datetime import datetime, timedelta from errno import ENOENT from functools import lru_cache from importlib import import_module from numbers import Integral, Number from threading import Lock from typing import Dict, Iterable, Mapping, Optional, Type, TypeVar from weakref import WeakValueDictionary from .core import get_deps K = TypeVar("K") V = TypeVar("V") system_encoding = sys.getdefaultencoding() if system_encoding == "ascii": system_encoding = "utf-8" def apply(func, args, kwargs=None): if kwargs: return func(*args, **kwargs) else: return func(*args) def _deprecated( *, version: str = None, message: str = None,
def truncate_words(s, num, end_text='...'): # truncate_words was removed in Django 1.5. truncate = end_text and ' %s' % end_text or '' return Truncator(s).words(num, truncate=truncate) truncate_words = allow_lazy(truncate_words, six.text_type) LTE_DJANGO_1_8 = django.VERSION < (1, 9) LTE_DJANGO_1_9 = django.VERSION < (1, 10) GTE_DJANGO_1_10 = django.VERSION >= (1, 10) if not six.PY3: fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() # copied from django.utils._os (not present in Django 1.4) def upath(path): """ Always return a unicode path. """ if six.PY2 and not isinstance(path, six.text_type): return path.decode(fs_encoding) return path # copied from django-cms (for compatibility with Django 1.4) try: from django.utils.encoding import force_unicode # flake8: noqa
# -*- coding:utf-8 -*- import sys, os default_encoding = 'utf-8' if sys.getdefaultencoding() != default_encoding: reload(sys) sys.setdefaultencoding(default_encoding) from django.db import models from django import forms from django.utils.html import format_html import uuid # Create your models here. class Catinfo(models.Model): name = models.CharField(max_length=10, verbose_name="名称") nameinfo = models.CharField(max_length=1000) feature = models.CharField(max_length=1000) livemethod = models.CharField(max_length=1000) feednn = models.CharField(max_length=1000) feedmethod = models.CharField(max_length=1000) def __str__(self): return self.name class Person(models.Model): sex_choice = ( (0, '男'), (1, '女'), )
def _earlyExceptionHandler(ty, value, traceback): util.ipmi_report(constants.IPMI_FAILED) util.vtActivate(1) return sys.__excepthook__(ty, value, traceback) sys.excepthook = _earlyExceptionHandler if conf.system.can_audit: # auditd will turn into a daemon and exit. Ignore startup errors try: util.execWithRedirect("/sbin/auditd", []) except OSError: pass log.info("anaconda called with cmdline = %s", sys.argv) log.info("Default encoding = %s ", sys.getdefaultencoding()) # start dbus session (if not already running) and run boss in it try: anaconda.dbus_launcher.start() except TimeoutError as e: stdout_log.error(str(e)) anaconda.dbus_launcher.stop() util.ipmi_report(constants.IPMI_ABORTED) time.sleep(10) sys.exit(1) # Collect all addon paths addon_paths = collect_addon_paths(constants.ADDON_PATHS) # If we were given a kickstart file on the command line, parse (but do not
import dataclasses import importlib.abc import importlib.util import itertools import logging import os import shutil import subprocess import sys import traceback import typing as t logger = logging.getLogger(__name__) SHELL = "/bin/bash" ENCODING = sys.getdefaultencoding() if t.TYPE_CHECKING or sys.version_info[:2] >= (3, 9): _T_CompletedProcess = subprocess.CompletedProcess[str] else: _T_CompletedProcess = subprocess.CompletedProcess _K = t.TypeVar("_K") _V = t.TypeVar("_V") def rm_singletons( d: t.Dict[_K, t.Union[_V, t.List[_V]]]) -> t.Dict[_K, t.List[_V]]: """Convert single values in a dictionary to a list with that value. >>> rm_singletons({ "k": "v" })
def fix_bin_or_dylib(self, fname): """Modifies the interpreter section of 'fname' to fix the dynamic linker, or the RPATH section, to fix the dynamic library search path This method is only required on NixOS and uses the PatchELF utility to change the interpreter/RPATH of ELF executables. Please see https://nixos.org/patchelf.html for more information """ default_encoding = sys.getdefaultencoding() try: ostype = subprocess.check_output( ['uname', '-s']).strip().decode(default_encoding) except subprocess.CalledProcessError: return except OSError as reason: if getattr(reason, 'winerror', None) is not None: return raise reason if ostype != "Linux": return # Use `/etc/os-release` instead of `/etc/NIXOS`. # The latter one does not exist on NixOS when using tmpfs as root. try: with open("/etc/os-release", "r") as f: if not any(line.strip() == "ID=nixos" for line in f): return except FileNotFoundError: return if os.path.exists("/lib"): return # At this point we're pretty sure the user is running NixOS nix_os_msg = "info: you seem to be running NixOS. Attempting to patch" print(nix_os_msg, fname) # Only build `.nix-deps` once. nix_deps_dir = self.nix_deps_dir if not nix_deps_dir: # Run `nix-build` to "build" each dependency (which will likely reuse # the existing `/nix/store` copy, or at most download a pre-built copy). # # Importantly, we create a gc-root called `.nix-deps` in the `build/` # directory, but still reference the actual `/nix/store` path in the rpath # as it makes it significantly more robust against changes to the location of # the `.nix-deps` location. # # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). # zlib: Needed as a system dependency of `libLLVM-*.so`. # patchelf: Needed for patching ELF binaries (see doc comment above). nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps") nix_expr = ''' with (import <nixpkgs> {}); symlinkJoin { name = "rust-stage0-dependencies"; paths = [ zlib patchelf stdenv.cc.bintools ]; } ''' try: subprocess.check_output([ "nix-build", "-E", nix_expr, "-o", nix_deps_dir, ]) except subprocess.CalledProcessError as reason: print("warning: failed to call nix-build:", reason) return self.nix_deps_dir = nix_deps_dir patchelf = "{}/bin/patchelf".format(nix_deps_dir) rpath_entries = [ # Relative default, all binary and dynamic libraries we ship # appear to have this (even when `../lib` is redundant). "$ORIGIN/../lib", os.path.join(os.path.realpath(nix_deps_dir), "lib") ] patchelf_args = ["--set-rpath", ":".join(rpath_entries)] if not fname.endswith(".so"): # Finally, set the corret .interp for binaries with open("{}/nix-support/dynamic-linker".format( nix_deps_dir)) as dynamic_linker: patchelf_args += [ "--set-interpreter", dynamic_linker.read().rstrip() ] try: subprocess.check_output([patchelf] + patchelf_args + [fname]) except subprocess.CalledProcessError as reason: print("warning: failed to call patchelf:", reason) return
def download_toolchain(self, stage0=True, rustc_channel=None): """Fetch the build system for Rust, written in Rust This method will build a cache directory, then it will fetch the tarball which has the stage0 compiler used to then bootstrap the Rust compiler itself. Each downloaded tarball is extracted, after that, the script will move all the content to the right place. """ if rustc_channel is None: rustc_channel = self.rustc_channel rustfmt_channel = self.rustfmt_channel bin_root = self.bin_root(stage0) key = self.date if not stage0: key += str(self.rustc_commit) if self.rustc(stage0).startswith(bin_root) and \ (not os.path.exists(self.rustc(stage0)) or self.program_out_of_date(self.rustc_stamp(stage0), key)): if os.path.exists(bin_root): shutil.rmtree(bin_root) tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' filename = "rust-std-{}-{}{}".format(rustc_channel, self.build, tarball_suffix) pattern = "rust-std-{}".format(self.build) self._download_component_helper(filename, pattern, tarball_suffix, stage0) filename = "rustc-{}-{}{}".format(rustc_channel, self.build, tarball_suffix) self._download_component_helper(filename, "rustc", tarball_suffix, stage0) # download-rustc doesn't need its own cargo, it can just use beta's. if stage0: filename = "cargo-{}-{}{}".format(rustc_channel, self.build, tarball_suffix) self._download_component_helper(filename, "cargo", tarball_suffix) self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root)) else: filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix) self._download_component_helper(filename, "rustc-dev", tarball_suffix, stage0) self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root)) self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root)) lib_dir = "{}/lib".format(bin_root) for lib in os.listdir(lib_dir): if lib.endswith(".so"): self.fix_bin_or_dylib(os.path.join(lib_dir, lib)) with output(self.rustc_stamp(stage0)) as rust_stamp: rust_stamp.write(key) if self.rustfmt() and self.rustfmt().startswith(bin_root) and ( not os.path.exists(self.rustfmt()) or self.program_out_of_date( self.rustfmt_stamp(), self.rustfmt_channel)): if rustfmt_channel: tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' [channel, date] = rustfmt_channel.split('-', 1) filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix) self._download_component_helper(filename, "rustfmt-preview", tarball_suffix, key=date) self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root)) self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root)) with output(self.rustfmt_stamp()) as rustfmt_stamp: rustfmt_stamp.write(self.rustfmt_channel) # Avoid downloading LLVM twice (once for stage0 and once for the master rustc) if self.downloading_llvm() and stage0: # We want the most recent LLVM submodule update to avoid downloading # LLVM more often than necessary. # # This git command finds that commit SHA, looking for bors-authored # merges that modified src/llvm-project or other relevant version # stamp files. # # This works even in a repository that has not yet initialized # submodules. top_level = subprocess.check_output([ "git", "rev-parse", "--show-toplevel", ]).decode(sys.getdefaultencoding()).strip() llvm_sha = subprocess.check_output([ "git", "rev-list", "[email protected]", "-n1", "--merges", "--first-parent", "HEAD", "--", "{}/src/llvm-project".format(top_level), "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level), # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` "{}/src/version".format(top_level) ]).decode(sys.getdefaultencoding()).strip() llvm_assertions = self.get_toml('assertions', 'llvm') == 'true' llvm_root = self.llvm_root() llvm_lib = os.path.join(llvm_root, "lib") if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)): self._download_ci_llvm(llvm_sha, llvm_assertions) for binary in ["llvm-config", "FileCheck"]: self.fix_bin_or_dylib( os.path.join(llvm_root, "bin", binary)) for lib in os.listdir(llvm_lib): if lib.endswith(".so"): self.fix_bin_or_dylib(os.path.join(llvm_lib, lib)) with output(self.llvm_stamp()) as llvm_stamp: llvm_stamp.write(llvm_sha + str(llvm_assertions))
def default_build_triple(verbose): """Build triple as in LLVM""" # If the user already has a host build triple with an existing `rustc` # install, use their preference. This fixes most issues with Windows builds # being detected as GNU instead of MSVC. default_encoding = sys.getdefaultencoding() try: version = subprocess.check_output(["rustc", "--version", "--verbose"], stderr=subprocess.DEVNULL) version = version.decode(default_encoding) host = next(x for x in version.split('\n') if x.startswith("host: ")) triple = host.split("host: ")[1] if verbose: print("detected default triple {}".format(triple)) return triple except Exception as e: if verbose: print("rustup not detected: {}".format(e)) print("falling back to auto-detect") required = sys.platform != 'win32' ostype = require(["uname", "-s"], exit=required) cputype = require(['uname', '-m'], exit=required) # If we do not have `uname`, assume Windows. if ostype is None or cputype is None: return 'x86_64-pc-windows-msvc' ostype = ostype.decode(default_encoding) cputype = cputype.decode(default_encoding) # The goal here is to come up with the same triple as LLVM would, # at least for the subset of platforms we're willing to target. ostype_mapper = { 'Darwin': 'apple-darwin', 'DragonFly': 'unknown-dragonfly', 'FreeBSD': 'unknown-freebsd', 'Haiku': 'unknown-haiku', 'NetBSD': 'unknown-netbsd', 'OpenBSD': 'unknown-openbsd' } # Consider the direct transformation first and then the special cases if ostype in ostype_mapper: ostype = ostype_mapper[ostype] elif ostype == 'Linux': os_from_sp = subprocess.check_output( ['uname', '-o']).strip().decode(default_encoding) if os_from_sp == 'Android': ostype = 'linux-android' else: ostype = 'unknown-linux-gnu' elif ostype == 'SunOS': ostype = 'pc-solaris' # On Solaris, uname -m will return a machine classification instead # of a cpu type, so uname -p is recommended instead. However, the # output from that option is too generic for our purposes (it will # always emit 'i386' on x86/amd64 systems). As such, isainfo -k # must be used instead. cputype = require(['isainfo', '-k']).decode(default_encoding) # sparc cpus have sun as a target vendor if 'sparc' in cputype: ostype = 'sun-solaris' elif ostype.startswith('MINGW'): # msys' `uname` does not print gcc configuration, but prints msys # configuration. so we cannot believe `uname -m`: # msys1 is always i686 and msys2 is always x86_64. # instead, msys defines $MSYSTEM which is MINGW32 on i686 and # MINGW64 on x86_64. ostype = 'pc-windows-gnu' cputype = 'i686' if os.environ.get('MSYSTEM') == 'MINGW64': cputype = 'x86_64' elif ostype.startswith('MSYS'): ostype = 'pc-windows-gnu' elif ostype.startswith('CYGWIN_NT'): cputype = 'i686' if ostype.endswith('WOW64'): cputype = 'x86_64' ostype = 'pc-windows-gnu' elif sys.platform == 'win32': # Some Windows platforms might have a `uname` command that returns a # non-standard string (e.g. gnuwin32 tools returns `windows32`). In # these cases, fall back to using sys.platform. return 'x86_64-pc-windows-msvc' else: err = "unknown OS type: {}".format(ostype) sys.exit(err) if cputype == 'powerpc' and ostype == 'unknown-freebsd': cputype = subprocess.check_output(['uname', '-p' ]).strip().decode(default_encoding) cputype_mapper = { 'BePC': 'i686', 'aarch64': 'aarch64', 'amd64': 'x86_64', 'arm64': 'aarch64', 'i386': 'i686', 'i486': 'i686', 'i686': 'i686', 'i786': 'i686', 'powerpc': 'powerpc', 'powerpc64': 'powerpc64', 'powerpc64le': 'powerpc64le', 'ppc': 'powerpc', 'ppc64': 'powerpc64', 'ppc64le': 'powerpc64le', 's390x': 's390x', 'x64': 'x86_64', 'x86': 'i686', 'x86-64': 'x86_64', 'x86_64': 'x86_64' } # Consider the direct transformation first and then the special cases if cputype in cputype_mapper: cputype = cputype_mapper[cputype] elif cputype in {'xscale', 'arm'}: cputype = 'arm' if ostype == 'linux-android': ostype = 'linux-androideabi' elif ostype == 'unknown-freebsd': cputype = subprocess.check_output( ['uname', '-p']).strip().decode(default_encoding) ostype = 'unknown-freebsd' elif cputype == 'armv6l': cputype = 'arm' if ostype == 'linux-android': ostype = 'linux-androideabi' else: ostype += 'eabihf' elif cputype in {'armv7l', 'armv8l'}: cputype = 'armv7' if ostype == 'linux-android': ostype = 'linux-androideabi' else: ostype += 'eabihf' elif cputype == 'mips': if sys.byteorder == 'big': cputype = 'mips' elif sys.byteorder == 'little': cputype = 'mipsel' else: raise ValueError("unknown byteorder: {}".format(sys.byteorder)) elif cputype == 'mips64': if sys.byteorder == 'big': cputype = 'mips64' elif sys.byteorder == 'little': cputype = 'mips64el' else: raise ValueError('unknown byteorder: {}'.format(sys.byteorder)) # only the n64 ABI is supported, indicate it ostype += 'abi64' elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64': pass else: err = "unknown cpu type: {}".format(cputype) sys.exit(err) return "{}-{}".format(cputype, ostype)
def prints(tstr): """ lovely unicode """ sys.stdout.write('%s\n' % (tstr.encode(sys.getdefaultencoding(), 'replace'))) sys.stdout.flush()
# coding:utf-8 # create by zhaotianxiang import pymysql import sys print(sys.getdefaultencoding()) from ConnectToMySQL import connectMySQL db = connectMySQL() # 后面的编码格式极其重要,耽误了两个小时,下次操作数据的时候一定要注意设置编码的一致 cursor = db.cursor() def createTable(): cursor.execute("DROP TABLE IF EXISTS DailyBoxOffices") sql = """CREATE TABLE ( itemID INT NOT NULL PRIMARY KEY AUTO_INCREMENT, itemDate DATE NOT NULL, movieName VARCHAR(45) NOT NULL, boxOffice CHAR(10), proportion CHAR(10), attendence CHAR(10), releaseDays CHAR(10))""" try: cursor.execute(sql) except: print("error to create table!\n") def storeToDatabase(): file_read = open('../data/movie_erverday_information.txt',
def onOpen(self, loadFirst='', loadEncode=''): """ 2.1: total rewrite for Unicode support; open in text mode with an encoding passed in, input from the user, in textconfig, or platform default, or open as binary bytes for arbitrary Unicode encodings as last resort and drop \r in Windows end-lines if present so text displays normally; content fetches are returned as str, so need to encode on saves: keep encoding used here; tests if file is okay ahead of time to try to avoid opens; we could also load and manually decode bytes to str to avoid multiple open attempts, but this is unlikely to try all cases; encoding behavior is configurable in the local textConfig.py: 1) tries known type first if passed in by client (email charsets) 2) if opensAskUser True, try user input next (prefill wih defaults) 3) if opensEncoding nonempty, try this encoding next: 'latin-1', etc. 4) tries sys.getdefaultencoding() platform default next 5) uses binary mode bytes and Tk policy as the last resort """ if self.text_edit_modified(): # 2.0 if not askyesno('PyEdit', 'Text has changed: discard changes?'): return file = loadFirst or self.my_askopenfilename() if not file: return if not os.path.isfile(file): showerror('PyEdit', 'Could not open file ' + file) return # try known encoding if passed and accurate (e.g., email) text = None # empty file = '' = False: test for None! if loadEncode: try: text = open(file, 'r', encoding=loadEncode).read() self.knownEncoding = loadEncode except (UnicodeError, LookupError, IOError): # lookup: bad name pass # try user input, prefill with next choice as default if text == None and self.opensAskUser: self.update() # else dialog doesn't appear in rare cases askuser = askstring('PyEdit', 'Enter Unicode encoding for open', initialvalue=(self.opensEncoding or sys.getdefaultencoding() or '')) if askuser: try: text = open(file, 'r', encoding=askuser).read() self.knownEncoding = askuser except (UnicodeError, LookupError, IOError): pass # try config file (or before ask user?) if text == None and self.opensEncoding: try: text = open(file, 'r', encoding=self.opensEncoding).read() self.knownEncoding = self.opensEncoding except (UnicodeError, LookupError, IOError): pass # try platform default (utf-8 on windows; try utf8 always?) if text == None: try: text = open(file, 'r', encoding=sys.getdefaultencoding()).read() self.knownEncoding = sys.getdefaultencoding() except (UnicodeError, LookupError, IOError): pass # last resort: use binary bytes and rely on Tk to decode if text == None: try: text = open(file, 'rb').read() # bytes for Unicode text = text.replace(b'\r\n', b'\n') # for display, saves self.knownEncoding = None except IOError: pass if text == None: showerror('PyEdit', 'Could not decode and open file ' + file) else: self.setAllText(text) self.setFileName(file) self.text.edit_reset() # 2.0: clear undo/redo stks self.text.edit_modified(0) # 2.0: clear modified flag
def onSaveAs(self, forcefile=None): """ 2.1: total rewrite for Unicode support: Text content is always returned as a str, so we must deal with encodings to save to a file here, regardless of open mode of the output file (binary requires bytes, and text must encode); tries the encoding used when opened or saved (if known), user input, config file setting, and platform default last; most users can use platform default; retains successful encoding name here for next save, because this may be the first Save after New or a manual text insertion; Save and SaveAs may both use last known enocding, per config file (it probably should be used for Save, but SaveAs usage is unclear); gui prompts are prefilled with the known encoding if there is one; does manual text.encode() to avoid creating file; text mode files perform platform specific end-line conversion: Windows \r dropped if present on open by text mode (auto) and binary mode (manually); if manual content inserts, must delete \r else duplicates here; knownEncoding=None before first Open or Save, after New, if binary Open; encoding behavior is configurable in the local textConfig.py: 1) if savesUseKnownEncoding > 0, try encoding from last open or save 2) if savesAskUser True, try user input next (prefill with known?) 3) if savesEncoding nonempty, try this encoding next: 'utf-8', etc 4) tries sys.getdefaultencoding() as a last resort """ filename = forcefile or self.my_asksaveasfilename() if not filename: return text = self.getAllText() # 2.1: a str string, with \n eolns, encpick = None # even if read/inserted as bytes # try known encoding at latest Open or Save, if any if self.knownEncoding and ( # enc known? (forcefile and self.savesUseKnownEncoding >= 1) or # on Save? (not forcefile and self.savesUseKnownEncoding >= 2)): # on SaveAs? try: text.encode(self.knownEncoding) encpick = self.knownEncoding except UnicodeError: pass # try user input, prefill with known type, else next choice if not encpick and self.savesAskUser: self.update() # else dialog doesn't appear in rare cases askuser = askstring('PyEdit', 'Enter Unicode encoding for save', initialvalue=(self.knownEncoding or self.savesEncoding or sys.getdefaultencoding() or '')) if askuser: try: text.encode(askuser) encpick = askuser except (UnicodeError, LookupError): # LookupError: bad name pass # UnicodeError: can't encode # try config file if not encpick and self.savesEncoding: try: text.encode(self.savesEncoding) encpick = self.savesEncoding except (UnicodeError, LookupError): pass # try platform default (utf8 on windows) if not encpick: try: text.encode(sys.getdefaultencoding()) encpick = sys.getdefaultencoding() except (UnicodeError, LookupError): pass # open in text mode for endlines + encoding if not encpick: showerror('PyEdit', 'Could not encode for file ' + filename) else: try: file = open(filename, 'w', encoding=encpick) file.write(text) file.close() except: showerror('PyEdit', 'Could not write file ' + filename) else: self.setFileName(filename) # may be newly created self.text.edit_modified(0) # 2.0: clear modified flag self.knownEncoding = encpick # 2.1: keep enc for next save
def test_getdefaultencoding(self): if test.test_support.have_unicode: self.assertRaises(TypeError, sys.getdefaultencoding, 42) # can't check more than the type, as the user might have changed it self.assertIsInstance(sys.getdefaultencoding(), str)
numbers = [1, 2, 3, 4, 11, 22, 33, 88, 99, 1000] print numbers even = [] odd = [] while len(numbers) > 0: num = numbers.pop() if (num % 2 == 0): even.append(num) else: odd.append(num) print even print odd print even + odd print set(even + odd) count = 0 while (count <= 9): print "the count is:", count count += 1 print "gooods bye" for str in "yanzhiwei": print str pass print "end for;" import sys print sys.getdefaultencoding() print isinstance("中国", unicode)
def _uni(s, e): # hack for py3 if sys.version_info[0] > 2: return str(bytes(s, sys.getdefaultencoding()), e) else: return unicode(s, e)
"NIEP": "http://eida-sc3.infp.ro", "NOA": "http://eida.gein.noa.gr", "ODC": "http://www.orfeus-eu.org", "ORFEUS": "http://www.orfeus-eu.org", "RESIF": "http://ws.resif.fr", "SCEDC": "http://service.scedc.caltech.edu", "USGS": "http://earthquake.usgs.gov", "USP": "http://sismo.iag.usp.br", } FDSNWS = ("dataselect", "event", "station") if PY2: platform_ = platform.platform().decode("ascii", "ignore") else: encoding = sys.getdefaultencoding() or "UTF-8" platform_ = platform.platform().encode(encoding).decode("ascii", "ignore") # The default User Agent that will be sent with every request. DEFAULT_USER_AGENT = "ObsPy/%s (%s, Python %s)" % (__version__, platform_, platform.python_version()) # The default parameters. Different services can choose to add more. It always # contains the long name first and the short name second. If it has no short # name, it is simply a tuple with only one entry. DEFAULT_DATASELECT_PARAMETERS = [ "starttime", "endtime", "network", "station", "location", "channel" ] OPTIONAL_DATASELECT_PARAMETERS = ["quality", "minimumlength", "longestonly"] DEFAULT_STATION_PARAMETERS = [
# coding=utf-8 import socket import sys import logging sys.getdefaultencoding() logging.basicConfig(format='%(levelname)s: %(message)s') def G(s): return "%s[32;2m%s%s[0m" % (chr(27), s, chr(27)) def A(s): return "%s[36;2m%s%s[0m" % (chr(27), s, chr(27)) def R(s): return "%s[31;2m%s%s[0m" % (chr(27), s, chr(27)) def tcp_test(url, port): """abcd""" sk = socket.socket() sk.settimeout(1) url, port = sys.argv[1], sys.argv[2] try: sk.connect((url, int(port))) except Exception: return 'Server(%s) port %s ' % (url, port) + R('not connect!') sk.close()
def get_gitlab_status(): return (subprocess.run( "docker inspect -f {{.State.Health.Status}} gitlab".split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, ).stdout.decode(sys.getdefaultencoding()).strip())
try: import imdb except ImportError: print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 2: print('Only one argument is required:') print(' %s "company name"' % sys.argv[0]) sys.exit(2) name = sys.argv[1] i = imdb.IMDb() out_encoding = sys.stdout.encoding or sys.getdefaultencoding() try: # Do the search, and get the results (a list of company objects). results = i.search_company(name) except imdb.IMDbError as e: print("Probably you're not connected to Internet. Complete error report:") print(e) sys.exit(3) # Print the results. print(' %s result%s for "%s":' % (len(results), ('', 's')[len(results) != 1], name)) print('companyID\t: imdbID : name') # Print the long imdb name for every company.
#!/usr/bin/python # -*- coding: utf-8 -*- import logging import os from xml.dom.minidom import parse from logging.handlers import TimedRotatingFileHandler import sys if sys.getdefaultencoding() != 'utf-8': reload(sys) sys.setdefaultencoding('utf-8') logger = logging.getLogger() logger.setLevel(logging.DEBUG) if not os.path.exists("logs"): os.mkdir("logs") handler = TimedRotatingFileHandler(filename="logs/upload.log", when="d", interval=1, backupCount=90) handler.setFormatter( logging.Formatter( '%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s - %(message)s' )) logger.addHandler(handler) ftp_param = {} upload_params = [] dom = parse("./config.xml") # 获取文件元素对象 document = dom.documentElement ftp_server = document.getElementsByTagName("ftp_server")[0]
def encode(self, encoding=sys.getdefaultencoding()): """More ``unicode`` class emulation.""" return str(self).encode(encoding)
def decodeCommandLine(self, cmdline): """Turn a byte string from the command line into a unicode string. """ codec = getattr(sys.stdin, 'encoding', None) or sys.getdefaultencoding() return unicode(cmdline, codec)
def handle_encoding(bot, ievent): """ show default encoding. """ ievent.reply('default encoding is %s' % bot.encoding or sys.getdefaultencoding())
def compile_cython_modules(profile=False, compile_more=False, cython_with_refnanny=False): source_root = os.path.abspath(os.path.dirname(__file__)) compiled_modules = [ "Cython.Plex.Scanners", "Cython.Plex.Actions", "Cython.Compiler.Lexicon", "Cython.Compiler.Scanning", "Cython.Compiler.Parsing", "Cython.Compiler.Visitor", "Cython.Compiler.FlowControl", "Cython.Compiler.Code", "Cython.Runtime.refnanny", # "Cython.Compiler.FusedNode", "Cython.Tempita._tempita", ] if compile_more: compiled_modules.extend([ "Cython.Build.Dependencies", "Cython.Compiler.ParseTreeTransforms", "Cython.Compiler.Nodes", "Cython.Compiler.ExprNodes", "Cython.Compiler.ModuleNode", "Cython.Compiler.Optimize", ]) from distutils.spawn import find_executable from distutils.sysconfig import get_python_inc pgen = find_executable( 'pgen', os.pathsep.join([ os.environ['PATH'], os.path.join(get_python_inc(), '..', 'Parser') ])) if not pgen: print("Unable to find pgen, not compiling formal grammar.") else: parser_dir = os.path.join(os.path.dirname(__file__), 'Cython', 'Parser') grammar = os.path.join(parser_dir, 'Grammar') subprocess.check_call([ pgen, os.path.join(grammar), os.path.join(parser_dir, 'graminit.h'), os.path.join(parser_dir, 'graminit.c'), ]) cst_pyx = os.path.join(parser_dir, 'ConcreteSyntaxTree.pyx') if os.stat(grammar)[stat.ST_MTIME] > os.stat(cst_pyx)[stat.ST_MTIME]: mtime = os.stat(grammar)[stat.ST_MTIME] os.utime(cst_pyx, (mtime, mtime)) compiled_modules.extend([ "Cython.Parser.ConcreteSyntaxTree", ]) defines = [] if cython_with_refnanny: defines.append(('CYTHON_REFNANNY', '1')) extensions = [] if sys.version_info[0] >= 3: from Cython.Distutils import build_ext as build_ext_orig for module in compiled_modules: source_file = os.path.join(source_root, *module.split('.')) if os.path.exists(source_file + ".py"): pyx_source_file = source_file + ".py" else: pyx_source_file = source_file + ".pyx" dep_files = [] if os.path.exists(source_file + '.pxd'): dep_files.append(source_file + '.pxd') if '.refnanny' in module: defines_for_module = [] else: defines_for_module = defines extensions.append( Extension(module, sources=[pyx_source_file], define_macros=defines_for_module, depends=dep_files)) class build_ext(build_ext_orig): # we must keep the original modules alive to make sure # their code keeps working when we remove them from # sys.modules dead_modules = [] def build_extensions(self): # add path where 2to3 installed the transformed sources # and make sure Python (re-)imports them from there already_imported = [ module for module in sys.modules if module == 'Cython' or module.startswith('Cython.') ] keep_alive = self.dead_modules.append for module in already_imported: keep_alive(sys.modules[module]) del sys.modules[module] sys.path.insert(0, os.path.join(source_root, self.build_lib)) if profile: from Cython.Compiler.Options import directive_defaults directive_defaults['profile'] = True print("Enabled profiling for the Cython binary modules") build_ext_orig.build_extensions(self) setup_args['ext_modules'] = extensions add_command_class("build_ext", build_ext) else: # Python 2.x from distutils.command.build_ext import build_ext as build_ext_orig try: class build_ext(build_ext_orig): def build_extension(self, ext, *args, **kargs): try: build_ext_orig.build_extension(self, ext, *args, **kargs) except StandardError: print("Compilation of '%s' failed" % ext.sources[0]) from Cython.Compiler.Main import compile from Cython import Utils if profile: from Cython.Compiler.Options import directive_defaults directive_defaults['profile'] = True print("Enabled profiling for the Cython binary modules") source_root = os.path.dirname(__file__) for module in compiled_modules: source_file = os.path.join(source_root, *module.split('.')) if os.path.exists(source_file + ".py"): pyx_source_file = source_file + ".py" else: pyx_source_file = source_file + ".pyx" c_source_file = source_file + ".c" source_is_newer = False if not os.path.exists(c_source_file): source_is_newer = True else: c_last_modified = Utils.modification_time(c_source_file) if Utils.file_newer_than(pyx_source_file, c_last_modified): source_is_newer = True else: pxd_source_file = source_file + ".pxd" if os.path.exists( pxd_source_file) and Utils.file_newer_than( pxd_source_file, c_last_modified): source_is_newer = True if source_is_newer: print("Compiling module %s ..." % module) result = compile(pyx_source_file) c_source_file = result.c_file if c_source_file: # Py2 distutils can't handle unicode file paths if isinstance(c_source_file, unicode): filename_encoding = sys.getfilesystemencoding() if filename_encoding is None: filename_encoding = sys.getdefaultencoding() c_source_file = c_source_file.encode(filename_encoding) if '.refnanny' in module: defines_for_module = [] else: defines_for_module = defines extensions.append( Extension(module, sources=[c_source_file], define_macros=defines_for_module)) else: print("Compilation failed") if extensions: setup_args['ext_modules'] = extensions add_command_class("build_ext", build_ext) except Exception: print(''' ERROR: %s Extension module compilation failed, looks like Cython cannot run properly on this system. To work around this, pass the option "--no-cython-compile". This will install a pure Python version of Cython without compiling its own sources. ''' % sys.exc_info()[1]) raise
def compile(**kwargs): """There are three modes of parameters :func:`compile()` can take: ``string``, ``filename``, and ``dirname``. The ``string`` parameter is the most basic way to compile SASS. It simply takes a string of SASS code, and then returns a compiled CSS string. :param string: SASS source code to compile. it's exclusive to ``filename`` and ``dirname`` parameters :type string: :class:`str` :param output_style: an optional coding style of the compiled result. choose one of: ``'nested'`` (default), ``'expanded'``, ``'compact'``, ``'compressed'`` :type output_style: :class:`str` :param source_comments: whether to add comments about source lines. :const:`False` by default :type source_comments: :class:`bool` :param include_paths: an optional list of paths to find ``@import``\ ed SASS/CSS source files :type include_paths: :class:`collections.Sequence` :param precision: optional precision for numbers. :const:`5` by default. :type precision: :class:`int` :param custom_functions: optional mapping of custom functions. see also below `custom functions <custom-functions_>`_ description :type custom_functions: :class:`collections.Set`, :class:`collections.Sequence`, :class:`collections.Mapping` :param indented: optional declaration that the string is SASS, not SCSS formatted. :const:`False` by default :type indented: :class:`bool` :returns: the compiled CSS string :param importers: optional callback functions. see also below `importer callbacks <importer-callbacks_>`_ description :type importers: :class:`collections.Callable` :rtype: :class:`str` :raises sass.CompileError: when it fails for any reason (for example the given SASS has broken syntax) The ``filename`` is the most commonly used way. It takes a string of SASS filename, and then returns a compiled CSS string. :param filename: the filename of SASS source code to compile. it's exclusive to ``string`` and ``dirname`` parameters :type filename: :class:`str` :param output_style: an optional coding style of the compiled result. choose one of: ``'nested'`` (default), ``'expanded'``, ``'compact'``, ``'compressed'`` :type output_style: :class:`str` :param source_comments: whether to add comments about source lines. :const:`False` by default :type source_comments: :class:`bool` :param source_map_filename: use source maps and indicate the source map output filename. :const:`None` means not using source maps. :const:`None` by default. :type source_map_filename: :class:`str` :param include_paths: an optional list of paths to find ``@import``\ ed SASS/CSS source files :type include_paths: :class:`collections.Sequence` :param precision: optional precision for numbers. :const:`5` by default. :type precision: :class:`int` :param custom_functions: optional mapping of custom functions. see also below `custom functions <custom-functions_>`_ description :type custom_functions: :class:`collections.Set`, :class:`collections.Sequence`, :class:`collections.Mapping` :param importers: optional callback functions. see also below `importer callbacks <importer-callbacks_>`_ description :type importers: :class:`collections.Callable` :returns: the compiled CSS string, or a pair of the compiled CSS string and the source map string if ``source_map_filename`` is set :rtype: :class:`str`, :class:`tuple` :raises sass.CompileError: when it fails for any reason (for example the given SASS has broken syntax) :raises exceptions.IOError: when the ``filename`` doesn't exist or cannot be read The ``dirname`` is useful for automation. It takes a pair of paths. The first of the ``dirname`` pair refers the source directory, contains several SASS source files to compiled. SASS source files can be nested in directories. The second of the pair refers the output directory that compiled CSS files would be saved. Directory tree structure of the source directory will be maintained in the output directory as well. If ``dirname`` parameter is used the function returns :const:`None`. :param dirname: a pair of ``(source_dir, output_dir)``. it's exclusive to ``string`` and ``filename`` parameters :type dirname: :class:`tuple` :param output_style: an optional coding style of the compiled result. choose one of: ``'nested'`` (default), ``'expanded'``, ``'compact'``, ``'compressed'`` :type output_style: :class:`str` :param source_comments: whether to add comments about source lines. :const:`False` by default :type source_comments: :class:`bool` :param include_paths: an optional list of paths to find ``@import``\ ed SASS/CSS source files :type include_paths: :class:`collections.Sequence` :param precision: optional precision for numbers. :const:`5` by default. :type precision: :class:`int` :param custom_functions: optional mapping of custom functions. see also below `custom functions <custom-functions_>`_ description :type custom_functions: :class:`collections.Set`, :class:`collections.Sequence`, :class:`collections.Mapping` :raises sass.CompileError: when it fails for any reason (for example the given SASS has broken syntax) .. _custom-functions: The ``custom_functions`` parameter can take three types of forms: :class:`~collections.Set`/:class:`~collections.Sequence` of \ :class:`SassFunction`\ s It is the most general form. Although pretty verbose, it can take any kind of callables like type objects, unnamed functions, and user-defined callables. .. code-block:: python sass.compile( ..., custom_functions={ sass.SassFunction('func-name', ('$a', '$b'), some_callable), ... } ) :class:`~collections.Mapping` of names to functions Less general, but easier-to-use form. Although it's not it can take any kind of callables, it can take any kind of *functions* defined using :keyword:`def`/:keyword:`lambda` syntax. It cannot take callables other than them since inspecting arguments is not always available for every kind of callables. .. code-block:: python sass.compile( ..., custom_functions={ 'func-name': lambda a, b: ..., ... } ) :class:`~collections.Set`/:class:`~collections.Sequence` of \ named functions Not general, but the easiest-to-use form for *named* functions. It can take only named functions, defined using :keyword:`def`. It cannot take lambdas sinc names are unavailable for them. .. code-block:: python def func_name(a, b): return ... sass.compile( ..., custom_functions={func_name} ) .. _importer-callbacks: Newer versions of ``libsass`` allow developers to define callbacks to be called and given a chance to process ``@import`` directives. You can define yours by passing in a list of callables via the ``importers`` parameter. The callables must be passed as 2-tuples in the form: .. code-block:: python (priority_int, callback_fn) A priority of zero is acceptable; priority determines the order callbacks are attempted. These callbacks must accept a single string argument representing the path passed to the ``@import`` directive, and either return ``None`` to indicate the path wasn't handled by that callback (to continue with others or fall back on internal ``libsass`` filesystem behaviour) or a list of one or more tuples, each in one of three forms: * A 1-tuple representing an alternate path to handle internally; or, * A 2-tuple representing an alternate path and the content that path represents; or, * A 3-tuple representing the same as the 2-tuple with the addition of a "sourcemap". All tuple return values must be strings. As a not overly realistic example: .. code-block:: python def my_importer(path): return [(path, '#' + path + ' { color: red; }')] sass.compile( ..., importers=[(0, my_importer)] ) Now, within the style source, attempting to ``@import 'button';`` will instead attach ``color: red`` as a property of an element with the imported name. .. versionadded:: 0.4.0 Added ``source_comments`` and ``source_map_filename`` parameters. .. versionchanged:: 0.6.0 The ``source_comments`` parameter becomes to take only :class:`bool` instead of :class:`str`. .. deprecated:: 0.6.0 Values like ``'none'``, ``'line_numbers'``, and ``'map'`` for the ``source_comments`` parameter are deprecated. .. versionadded:: 0.7.0 Added ``precision`` parameter. .. versionadded:: 0.7.0 Added ``custom_functions`` parameter. .. versionadded:: 0.11.0 ``source_map_filename`` no longer implies ``source_comments``. """ modes = set() for mode_name in MODES: if mode_name in kwargs: modes.add(mode_name) if not modes: raise TypeError('choose one at least in ' + and_join(MODES)) elif len(modes) > 1: raise TypeError( and_join(modes) + ' are exclusive each other; ' 'cannot be used at a time') precision = kwargs.pop('precision', 5) output_style = kwargs.pop('output_style', 'nested') if not isinstance(output_style, string_types): raise TypeError('output_style must be a string, not ' + repr(output_style)) try: output_style = OUTPUT_STYLES[output_style] except KeyError: raise CompileError('{0} is unsupported output_style; choose one of {1}' ''.format(output_style, and_join(OUTPUT_STYLES))) source_comments = kwargs.pop('source_comments', False) if source_comments in SOURCE_COMMENTS: if source_comments == 'none': deprecation_message = ('you can simply pass False to ' "source_comments instead of 'none'") source_comments = False elif source_comments in ('line_numbers', 'default'): deprecation_message = ('you can simply pass True to ' "source_comments instead of " + repr(source_comments)) source_comments = True else: deprecation_message = ("you don't have to pass 'map' to " 'source_comments but just need to ' 'specify source_map_filename') source_comments = False warnings.warn( "values like 'none', 'line_numbers', and 'map' for " 'the source_comments parameter are deprecated; ' + deprecation_message, DeprecationWarning) if not isinstance(source_comments, bool): raise TypeError('source_comments must be bool, not ' + repr(source_comments)) fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() def _get_file_arg(key): ret = kwargs.pop(key, None) if ret is not None and not isinstance(ret, string_types): raise TypeError('{} must be a string, not {!r}'.format(key, ret)) elif isinstance(ret, text_type): ret = ret.encode(fs_encoding) if ret and 'filename' not in modes: raise CompileError( '{} is only available with filename= keyword argument since ' 'has to be aware of it'.format(key)) return ret source_map_filename = _get_file_arg('source_map_filename') output_filename_hint = _get_file_arg('output_filename_hint') # #208: cwd is always included in include paths include_paths = (os.getcwd(), ) include_paths += tuple(kwargs.pop('include_paths', ()) or ()) include_paths = os.pathsep.join(include_paths) if isinstance(include_paths, text_type): include_paths = include_paths.encode(fs_encoding) custom_functions = kwargs.pop('custom_functions', ()) if isinstance(custom_functions, collections.Mapping): custom_functions = [ SassFunction.from_lambda(name, lambda_) for name, lambda_ in custom_functions.items() ] elif isinstance(custom_functions, (collections.Set, collections.Sequence)): custom_functions = [ func if isinstance(func, SassFunction) else SassFunction.from_named_function(func) for func in custom_functions ] else: raise TypeError( 'custom_functions must be one of:\n' '- a set/sequence of {0.__module__}.{0.__name__} objects,\n' '- a mapping of function name strings to lambda functions,\n' '- a set/sequence of named functions,\n' 'not {1!r}'.format(SassFunction, custom_functions)) importers = _validate_importers(kwargs.pop('importers', None)) if 'string' in modes: string = kwargs.pop('string') if isinstance(string, text_type): string = string.encode('utf-8') indented = kwargs.pop('indented', False) if not isinstance(indented, bool): raise TypeError('indented must be bool, not ' + repr(source_comments)) _check_no_remaining_kwargs(compile, kwargs) s, v = _sass.compile_string( string, output_style, source_comments, include_paths, precision, custom_functions, indented, importers, ) if s: return v.decode('utf-8') elif 'filename' in modes: filename = kwargs.pop('filename') if not isinstance(filename, string_types): raise TypeError('filename must be a string, not ' + repr(filename)) elif not os.path.isfile(filename): raise IOError('{0!r} seems not a file'.format(filename)) elif isinstance(filename, text_type): filename = filename.encode(fs_encoding) _check_no_remaining_kwargs(compile, kwargs) s, v, source_map = _sass.compile_filename( filename, output_style, source_comments, include_paths, precision, source_map_filename, custom_functions, importers, output_filename_hint, ) if s: v = v.decode('utf-8') if source_map_filename: source_map = source_map.decode('utf-8') v = v, source_map return v elif 'dirname' in modes: try: search_path, output_path = kwargs.pop('dirname') except ValueError: raise ValueError('dirname must be a pair of (source_dir, ' 'output_dir)') _check_no_remaining_kwargs(compile, kwargs) s, v = compile_dirname( search_path, output_path, output_style, source_comments, include_paths, precision, custom_functions, importers, ) if s: return else: raise TypeError('something went wrong') assert not s raise CompileError(v)
#-*- coding: gbk -*- # encoding =gbk import sys #sys.setdefaultencoding('utf-8') if sys.getdefaultencoding() != 'gbk': reload(sys) sys.setdefaultencoding('gbk') import os import gzip import time def read_gz_file(path): if os.path.exists(path): with gzip.open(path, 'rb') as pf: for line in pf: yield line else: print('the path [{}] is not exist!'.format(path)) def get_gz_fileName(path): if os.path.exists(path): gz_fileName = list() for root, dirs, files in os.walk(path): for gzfile in files: if os.path.splitext(gzfile)[1] == '.gz': gz_fileName.append(gzfile) return gz_fileName
def get_best_encoding(stream): """Returns the default stream encoding if not found.""" rv = getattr(stream, "encoding", None) or sys.getdefaultencoding() if is_ascii_encoding(rv): return "utf-8" return rv
def __str__(self): enc = sys.stdin.encoding or sys.getdefaultencoding() return self.value.encode(enc, "replace")
def get_filesystem_encoding(): return sys.getfilesystemencoding() or sys.getdefaultencoding()
import hashlib from pathlib import Path from sys import getdefaultencoding from tempfile import NamedTemporaryFile import pytest FIXTURES_ROOT = Path(__file__).absolute().parent.joinpath('fixtures') SOURCE_FILE_NAME = 'data.csv' DESTINATION_FILE_NAME = 'data.json' CSV_ENCODING = getdefaultencoding() SOURCE_FILE_PATH = FIXTURES_ROOT.joinpath(SOURCE_FILE_NAME) DESTINATION_FILE_PATH = FIXTURES_ROOT.joinpath(DESTINATION_FILE_NAME) @pytest.fixture def data_csv_path(): return SOURCE_FILE_PATH @pytest.fixture def data_json_path(): return DESTINATION_FILE_PATH @pytest.fixture def named_temporary_file(): return NamedTemporaryFile(mode='wt')