def download_and_repack(country_code=None, network_code=None, circle=None, token=None, source=None, destination=None, byte_order="b", verbose=False): """ Downloads and packs the base station data. So far, only opencellid.org supported. Args: country_code (int): the country code; network_code (int): the network code; circle (tuple): latitude, longitude (degrees) and radius in km; token (str): service token; source (str): downloaded file name; destination (str): destination file name; mnc_block_size (int): the size of the mnc block; mcc_block_size (int): the size of the mcc block; byte_order (str): byte order; verbose (bool): prints verbose output; """ def v(*args, **kwargs): if verbose: print(*args, **kwargs) byte_order = byte_order.lower() if byte_order not in "bl": raise ValueError("Unknown byte order: {}".format(byte_order)) byte_order = dict(b=">", l="<")[byte_order] if destination is None: if country_code is None: destination = "all.bin" else: if network_code is None: destination = "{country_code}.bin".format( country_code=country_code) else: destination = "{country_code}-{network_code}.bin".format( country_code=country_code, network_code=network_code) v("Target: {}".format(destination)) if source is None: if token is None: v("No token specified: downloading from git") if country_code is None: raise ValueError("Cannot download worldwide database yet") else: url = "https://github.com/pulkin/agps-data/raw/master/opencellid.org/{country_code}.csv.gz".format( country_code=country_code) else: if country_code is None: url = "https://opencellid.org/ocid/downloads?token={token}&type=full&file=cell_towers.csv.gz".format( token=token) else: url = "https://opencellid.org/ocid/downloads?token={token}&type=mcc&file={country_code}.csv.gz".format( token=token, country_code=country_code) v("Downloading {} ...".format(url)) response = urllib.request.urlopen( urllib.request.Request( url, headers={ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36' })) buf = io.BytesIO(response.read()) else: v("Reading {} ...".format(source)) buf = open(source, 'rb') v("Unzipping ...") buf_raw = gzip.GzipFile(fileobj=buf, mode='rb') buf_txt = io.TextIOWrapper(buf_raw) v("Parsing ...") dtype = [ ('radio_type', 'S4'), ('mcc', byte_order + 'u2'), ('mnc', byte_order + 'u2'), ('area_code', byte_order + 'u2'), ('cell', byte_order + 'u2'), ('lon', byte_order + 'f4'), ('lat', byte_order + 'f4'), ] data = numpy.genfromtxt(buf_txt, dtype=dtype, skip_header=1, delimiter=",", usecols=(0, 1, 2, 3, 4, 6, 7)) v("Filtering ...") data = data[data["radio_type"] == b"GSM"][[ "mcc", "mnc", "area_code", "cell", "lon", "lat" ]] if country_code is not None: v(" - mcc: {:d}".format(len(data)), end="") data = data[data["mcc"] == country_code] v(" -> {:d}".format(len(data))) if network_code is not None: v(" - mnc: {:d}".format(len(data)), end="") data = data[data["mnc"] == network_code] v(" -> {:d}".format(len(data))) if circle is not None: v(" - circle: {:d}".format(len(data)), end="") theta0, phi0, r0 = circle theta = (data["lat"] - theta0) * numpy.pi / 180 phi = (data["lon"] - phi0) * numpy.pi / 180 mask = (theta**2 + phi**2 * numpy.cos(theta0)**2) < (r0 * 1e3 / earth_radius)**2 data = data[mask] v(" -> {:d}".format(len(data))) if len(data) == 0: raise ValueError("No data to save") v("Sorting ...") data = numpy.sort(data, order=("mcc", "mnc", "area_code", "cell")) v("Items total: {:d}".format(len(data))) v("Preparing tables ...") keys = "mcc", "mnc" mask = numpy.zeros(len(data), dtype=bool) mask[0] = True for k in keys: mask[1:] |= data[k][1:] != data[k][:-1] table_ptrs = numpy.where(mask)[0] table_data = recfunctions.repack_fields(data[table_ptrs][list(keys)]) v("Saving ...") with open(destination, 'wb') as f: f.write(b'agps-bin') f.write(b'\x00') f.write({">": b">", "<": b"<"}[byte_order]) f.write(struct.pack(byte_order + "L", len(table_ptrs))) for _d, _p in zip(table_data, table_ptrs): f.write(struct.pack(byte_order + "HHL", *_d, _p)) recfunctions.repack_fields(data[["area_code", "cell", "lon", "lat"]]).tofile(f) v("Total size: {:d} bytes".format(f.tell())) v("Done")
segment.strip() for split in splits[:-1] for segment in [split, glossary] if segment != '' ] return segments + [splits[-1].strip() ] if splits[-1] != '' else segments if __name__ == '__main__': # python 2/3 compatibility if sys.version_info < (3, 0): sys.stderr = codecs.getwriter('UTF-8')(sys.stderr) sys.stdout = codecs.getwriter('UTF-8')(sys.stdout) sys.stdin = codecs.getreader('UTF-8')(sys.stdin) else: sys.stdin = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8') sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8') sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', write_through=True, line_buffering=True) parser = create_parser() args = parser.parse_args() # read/write files as UTF-8 args.codes = codecs.open(args.codes.name, encoding='utf-8') if args.input.name != '<stdin>': args.input = codecs.open(args.input.name, encoding='utf-8') if args.output.name != '<stdout>': args.output = codecs.open(args.output.name, 'w', encoding='utf-8')
import urllib.request from urllib.parse import urlparse import sys import io sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding = 'utf-8') sys.stderr = io.TextIOWrapper(sys.stderr.detach(), encoding = 'utf-8') url = "http://www.encar.com/" mem = urllib.request.urlopen(url) print(type(mem)) print("geturl :",mem.geturl()) #결과 : http://www.encar.com/index.do print("status :",mem.status) #응답코드 200 (정상), 404(요청페이지 없음), 403 (Reject), 500 (서버 에러) print("headers :",mem.getheaders()) print("info :",mem.info(),"\n") #== headers print("getcode :",mem.getcode()) #== status print("read :",mem.read(10).decode('utf-8')) #해당 숫자만큼 불러옴 print(urlparse('http://www.encar.co.kr?test=test').query)
def create(): fa = fs.open_input_file(path) fp = FileProxy(fa, path, lambda: fs.open_input_file(path)) return io.TextIOWrapper(fp, encoding=encoding)
def _json_decode(self, json_bytes, encoding): tiow = io.TextIOWrapper(io.BytesIO(json_bytes), encoding=encoding, newline="") obj = json.load(tiow) tiow.close() return obj
def write_data_export_zip(self, request, response): with zipfile.ZipFile(response, mode="w") as zip_file: with zip_file.open(f"{self.slug}.json", mode="w") as json_binary_file, io.TextIOWrapper( json_binary_file) as json_file: json.dump( { "email": self.user.email, "date_joined": self.user.date_joined.isoformat(), "last_login": self.user.last_login.isoformat(), "url": request.build_absolute_uri(self.get_absolute_url()), "is_public": self.is_public, "is_approved": self.is_approved, "name": self.name, "city_or_town": self.city_or_town, "country": self.country, "cause_areas": list(map(CauseArea.label, self.cause_areas)), "cause_areas_other": self.cause_areas_other, "available_to_volunteer": self.available_to_volunteer, "open_to_job_offers": self.open_to_job_offers, "expertise_areas": list(map(ExpertiseArea.label, self.expertise_areas)), "expertise_areas_other": self.expertise_areas_other, "career_interest_areas": list( map(ExpertiseArea.label, self.career_interest_areas)), "available_as_speaker": self.available_as_speaker, "topics_i_speak_about": self.topics_i_speak_about, "organisational_affiliations": list( map( OrganisationalAffiliation.label, self.organisational_affiliations, )), "summary": self.summary, "giving_pledges": list(map(GivingPledge.label, self.giving_pledges)), "member_of_local_groups": [ request.build_absolute_uri( local_group.get_absolute_uri()) for local_group in self.local_groups.all() ], "organiser_of_local_groups": [ request.build_absolute_uri( local_group.get_absolute_uri()) for local_group in self.user.localgroup_set.all() ], "aliases": [ request.build_absolute_uri( urls.reverse("profile", kwargs={"slug": slug.slug})) for slug in self.slugs.filter(redirect=True) ], "legacy_hub_url": (self.legacy_record and request.build_absolute_uri( urls.reverse( "profile_legacy", kwargs={"legacy_record": self.legacy_record}, ))), }, json_file, indent=2, ) if self.image: with self.image.open() as image_src_file, zip_file.open( self.slug + pathlib.PurePath(self.image.name).suffix, mode="w") as image_dst_file: shutil.copyfileobj(image_src_file, image_dst_file)
def __init__(self, executable, configuration, model, analysis_threads=10, search_threads=1, output=False): self.output = output if output: print(' Launching KataGo...') def read_stream(name, stream, type_, buffer): if output: print(f' {name} thread has begun.') while True: line = stream.readline().rstrip() if (not self._ready and type_ is LineType.error and line.endswith( 'Started, ready to begin handling requests')): self._ready = True if output: print(' KataGo is ready to accept inputs.') if output: print(f' {name} read: {line}') if line: buffer.append((type_, line)) else: if output: print(f' {name} thread has finished.') break command = f'{executable} analysis -config {configuration} -model {model} -analysis-threads {analysis_threads} ' \ f'-override-config numSearchThreads={search_threads}' self._process = subprocess.Popen(command, stderr=subprocess.PIPE, stdin=subprocess.PIPE, stdout=subprocess.PIPE) self._buffer = [] self._ready = False stderr = io.TextIOWrapper(self._process.stderr, encoding='utf-8', errors='strict') self._errors = Thread(target=read_stream, args=('ERR', stderr, LineType.error, self._buffer)) self._errors.daemon = True self._errors.start() stdout = io.TextIOWrapper(self._process.stdout, encoding='utf-8', errors='strict') self._outputs = Thread(target=read_stream, args=('OUT', stdout, LineType.output, self._buffer)) self._outputs.daemon = True self._outputs.start() if output: print(' KataGo has launched.')
def textblock(filename, start, end, compression=None, encoding=system_encoding, linesep=os.linesep, buffersize=4096): """Pull out a block of text from a file given start and stop bytes. This gets data starting/ending from the next linesep delimiter. Each block consists of bytes in the range [start,end[, i.e. the stop byte is excluded. If `start` is 0, then `start` corresponds to the true start byte. If `start` is greater than 0 and does not point to the beginning of a new line, then `start` is incremented until it corresponds to the start byte of the next line. If `end` does not point to the beginning of a new line, then the line that begins before `end` is included in the block although its last byte exceeds `end`. Examples -------- >> with open('myfile.txt', 'wb') as f: .. f.write('123\n456\n789\nabc') In the example below, 1 and 10 don't line up with endlines. >> u''.join(textblock('myfile.txt', 1, 10)) '456\n789\n' """ # Make sure `linesep` is not a byte string because # `io.TextIOWrapper` in Python versions other than 2.7 dislike byte # strings for the `newline` argument. linesep = str(linesep) # Get byte representation of the line separator. bin_linesep = get_bin_linesep(encoding, linesep) bin_linesep_len = len(bin_linesep) if buffersize < bin_linesep_len: error = ('`buffersize` ({0:d}) must be at least as large as the ' 'number of line separator bytes ({1:d}).') raise ValueError(error.format(buffersize, bin_linesep_len)) chunksize = end - start with open(filename, 'rb', compression) as f: with io.BufferedReader(f) as fb: # If `start` does not correspond to the beginning of the file, we # need to move the file pointer to `start - len(bin_linesep)`, # search for the position of the next a line separator, and set # `start` to the position after that line separator. if start > 0: # `start` is decremented by `len(bin_linesep)` to detect the # case where the original `start` value corresponds to the # beginning of a line. start = max(0, start - bin_linesep_len) # Set the file pointer to `start`. fb.seek(start) # Number of bytes to shift the file pointer before reading a # new chunk to make sure that a multi-byte line separator, that # is split by the chunk reader, is still detected. shift = 1 - bin_linesep_len while True: buf = f.read(buffersize) if len(buf) < bin_linesep_len: raise StopIteration try: # Find the position of the next line separator and add # `len(bin_linesep)` which yields the position of the # first byte of the next line. start += buf.index(bin_linesep) start += bin_linesep_len except ValueError: # No line separator was found in the current chunk. # Before reading the next chunk, we move the file # pointer back `len(bin_linesep) - 1` bytes to make # sure that a multi-byte line separator, that may have # been split by the chunk reader, is still detected. start += len(buf) start += shift fb.seek(shift, os.SEEK_CUR) else: # We have found the next line separator, so we need to # set the file pointer to the first byte of the next # line. fb.seek(start) break with io.TextIOWrapper(fb, encoding, newline=linesep) as fbw: # Retrieve and yield lines until the file pointer reaches # `end`. while start < end: line = next(fbw) # We need to encode the line again to get the byte length # in order to correctly update `start`. bin_line_len = len(line.encode(encoding)) if chunksize < bin_line_len: error = ('`chunksize` ({0:d}) is less than the line ' 'length ({1:d}). This may cause duplicate ' 'processing of this line. It is advised to ' 'increase `chunksize`.') raise IOError(error.format(chunksize, bin_line_len)) yield line start += bin_line_len
def main(argv=None): if argv is None: argv = sys.argv if len(argv) < 3: sys.exit('Usage: ' + argv[0] + ' <results-xml> [<witness-xml>]* [--no-overwrite-status].\n') resultFile = argv[1] witnessFiles = [] isOverwrite = True for i in range(2, len(argv)): if len(argv) > i and not argv[i].startswith('--'): witnessFiles.append(argv[i]) if argv[i] == '--no-overwrite-status': isOverwrite = False if not os.path.exists(resultFile) or not os.path.isfile(resultFile): sys.exit('File {0} does not exist.'.format(repr(resultFile))) resultXML = tablegenerator.parse_results_file(resultFile) witnessSets = [] for witnessFile in witnessFiles: if not os.path.exists(witnessFile) or not os.path.isfile(witnessFile): sys.exit('File {0} does not exist.'.format(repr(witnessFile))) witnessXML = tablegenerator.parse_results_file(witnessFile) witnessSets.append(getWitnesses(witnessXML)) resultXML.set( 'options', '' + resultXML.get('options', default='') + ' [[ ' + witnessXML.get('options', default='') + ' ]]') resultXML.set( 'date', '' + resultXML.get('date', default='') + ' [[ ' + witnessXML.get('date', default='') + ' ]]') for result in resultXML.findall('run'): run = result.get('name') basename = os.path.basename(run) if 'correct' == result.findall('column[@title="category"]')[0].get( 'value'): statusVer = result.findall('column[@title="status"]')[0] categoryVer = result.findall('column[@title="category"]')[0] properties = result.get('properties').split(' ') expected_result = benchexec.result.satisfies_file_property( basename, properties) statusWit, categoryWit = (None, None) i = 0 for witnessSet in witnessSets: i = i + 1 witness = witnessSet.get(run, None) # copy data from witness if witness is not None: for column in witness: newColumn = ET.Element( 'column', { 'title': 'wit' + str(i) + '_' + column.get('title'), 'value': column.get('value'), 'hidden': column.get('hidden', 'false') }) result.append(newColumn) witnessSet.pop(run) statusWitNew, categoryWitNew = getWitnessResult( witness, expected_result) if expected_result == False: if statusWitNew.startswith( 'false(') or statusWit is None: statusWit, categoryWit = (statusWitNew, categoryWitNew) if expected_result == True: if statusWitNew.startswith( 'true') or statusWit is None: statusWit, categoryWit = (statusWitNew, categoryWitNew) # Overwrite status with status from witness if isOverwrite and statusWit is not None and categoryWit is not None: result.findall('column[@title="status"]')[0].set( 'value', statusWit) result.findall('column[@title="category"]')[0].set( 'value', categoryWit) # Clean-up an entry that can be inferred by table-generator automatically, avoids path confusion del result.attrib['logfile'] filename = resultFile + '.merged.xml.bz2' print(' ' + filename) open_func = bz2.BZ2File if hasattr(bz2.BZ2File, 'writable') else util.BZ2FileHack with io.TextIOWrapper(open_func(filename, 'wb'), encoding='utf-8') as xml_file: xml_file.write( xml_to_string(resultXML).replace(' \n', '').replace(' \n', ''))
#!/usr/bin/python import gzip import io ################################################################################# SRR3879604 # set dictionary and counters to empty or 0 bc_dic604 = {} total_bc_count604 = 0 dic_count604 = 0 total_repeat_count604 = 0 no_bc_count604 = 0 # open with gzip, make sure to use 'rt' with io.TextIOWrapper( io.BufferedReader( gzip.open( '/projectnb/bf528/project_4_scrnaseq/fastq/SRR3879604/SRR3879604_1_bc.fastq.gz' ))) as sample604: # loop through each line, gather barcodes, count barcode frequency, store count for non-bc samples too for line in sample604: if line[0] == '@': next_line = next(sample604) barcode = next_line[0:19] if barcode in bc_dic604: total_bc_count604 += 1 total_repeat_count604 += 1 bc_dic604[barcode] += 1 if barcode not in bc_dic604: total_bc_count604 += 1 dic_count604 += 1 bc_dic604[barcode] = 1 else:
def autoconstruct(self): # If camera device set in config use it, otherwise autodetect cameradev = None devicepaths = glob.glob("/dev/video*") if self.config.args.camera_device: self.logger.debug('camera_device specified: {}'.format(self.config.args.camera_device)) cameradev = self.config.args.camera_device else: # device not set, carry on and try to autodetect for devicepath in sorted(devicepaths): if not cameradev and self.check_input(devicepath): cameradev = devicepath self.logger.info('v4l2 device '+devicepath+' is a camera, autoselecting') elif not cameradev: self.logger.debug('v4l2 device '+devicepath+' is not a camera, ignoring') if not cameradev: raise ValueError('Error detecting camera video device') # Check the camera has a valid input try: self.vd = io.TextIOWrapper(open(cameradev, "r+b", buffering=0)) cp = v4l2.v4l2_capability() except Exception as e: raise ValueError("Camera not specified in config, or camera not valid: {}".format(repr(e))) if not self.check_input(): raise ValueError('Specified camera not valid') # Log info self.camera_info() # Try and autodetect Jetson/Tegra CSI connection if self.driver == 'tegra-video': self.logger.info('Nvidia Jetson/Tegra CSI connection detected, switching to nvarguscamerasrc') self.input = "nvarguscamerasrc" elif 'input' not in self.config.args or not self.config.args.input: self.input = "v4l2src" else: self.input = self.config.args.input # Try and autodetect MFC device self.mfcdev = None for devicepath in devicepaths: dp = io.TextIOWrapper(open(devicepath, "r+b", buffering=0)) ioctl(dp, v4l2.VIDIOC_QUERYCAP, cp) if cp.card == "s5p-mfc-enc": self.mfcdev = dp self.logger.info(f'MFC Hardware encoder detected, autoselecting {devicepath}') # If format set in config use it, otherwise autodetect streamtype = None if self.config.args.format: streamtype = self.config.args.format else: if self.input == "nvarguscamerasrc": self.logger.info('Nvidia Jetson/Tegra input detected, forcing Tegra stream format') streamtype = 'tegra' elif re.search("C920", self.card): self.logger.info("Logitech C920 detected, forcing H264 passthrough") streamtype = 'h264' # format not set, carry on and try to autodetect elif self.check_format('yuv'): self.logger.info('Camera YUV stream available, using yuv stream') streamtype = 'yuv' # Otherwise, check for an mjpeg->h264 encoder pipeline. elif self.check_format('mjpeg'): self.logger.info('Camera MJPEG stream available, using mjpeg stream') streamtype = 'mjpeg' # Lastly look for a h264 stream elif self.check_format('h264'): self.logger.info('Camera H264 stream available, using H264 stream') streamtype = 'h264' if not streamtype: raise ValueError('Error detecting camera video format') # If encoder set in config use it, otherwise set to h264 encoder = None if self.config.args.encoder: encoder = self.config.args.encoder if not encoder: encoder = "h264" self.logger.debug("Using encoder: {}".format(encoder)) # If raspberry camera detected set pixelformat to I420, otherwise set to YUY2 by default pixelformat = "YUY2" ioctl(self.vd, v4l2.VIDIOC_QUERYCAP, cp) if cp.driver == "bm2835 mmal": self.logger.info("Raspberry Pi Camera detected, setting pixel format to I420") pixelformat = "I420" # If raw pixelformat set in config override the defaults if 'pixelformat' in self.config.args and self.config.args.pixelformat: pixelformat = self.config.args.pixelformat self.logger.debug("Using pixelformat: {}".format(pixelformat)) # Create and start the stream try: self.logger.info("Creating stream object - device: {}, stream: {}, pixelformat: {}, encoder: {}, input: {}".format(cameradev, streamtype, pixelformat, encoder, self.input)) Streamer(self.config, streamtype, pixelformat, encoder, self.input, cameradev) if self.zeroconf: # Update the stream advertisement with the new info self.zeroconf.update({"stream":"replace_with_stream_info"}) except Exception as e: if self.zeroconf: self.zeroconf.update({"stream":""}) raise ValueError('Error creating {} stream: {}'.format(streamtype, repr(e))) # Inform systemd that start is complete self.logger.info("Notifying systemd of startup completion") self.notify.notify("READY=1") self.notify.notify("STATUS=Automatic Pipeline Initialisation Complete") while not self._should_shutdown: time.sleep(1)
def __iter__(self): wrapper = io.TextIOWrapper(self.file) yield from wrapper
def test_gzip_reader_textiowrapper(gzip_reader): with gzip_reader("tests/file.txt.gz", "rb") as f: wrapped = io.TextIOWrapper(f) assert wrapped.read() == CONTENT
def main(): # Get Python routines for reading part description/CSV files. readers = scan_for_readers() parser = ap.ArgumentParser( description="Generate single & multi-unit schematic symbols for KiCad from a CSV file." ) parser.add_argument( "-v", "--version", action="version", version="KiPart " + __version__ ) parser.add_argument( "input_files", nargs="+", type=str, metavar="file.[csv|txt|xlsx|zip]", help="Files for parts in CSV/text/Excel format or as such files in .zip archives.", ) parser.add_argument( "-r", "--reader", nargs="?", type=lambda s: unicode(s).lower(), choices=readers.keys(), default="generic", help="Name of function for reading the CSV or part description files.", ) parser.add_argument( "-s", "--sort", nargs="?", # type=str.lower, type=lambda s: unicode(s).lower(), choices=["row", "num", "name"], default="row", help="Sort the part pins by their entry order in the CSV file, their pin number, or their pin name.", ) parser.add_argument( "--reverse", action="store_true", help="Sort pins in reverse order." ) parser.add_argument( "--side", nargs="?", # type=str.lower, type=lambda s: unicode(s).lower(), choices=["left", "right", "top", "bottom"], default="left", help="Which side to place the pins by default.", ) parser.add_argument( "-o", "--output", nargs="?", type=str, metavar="file.lib", help="Generated KiCad symbol library for parts.", ) parser.add_argument( "-f", "--fuzzy_match", action="store_true", help="Use approximate string matching when looking-up the pin type, style and orientation.", ) parser.add_argument( "-b", "--bundle", action="store_true", help="Bundle multiple, identically-named power and ground pins each into a single schematic pin.", ) parser.add_argument( "-a", "--append", "--add", action="store_true", help="Add parts to an existing part library. Overwrite existing parts only if used in conjunction with -w.", ) parser.add_argument( "-w", "--overwrite", action="store_true", help="Allow overwriting of an existing part library.", ) parser.add_argument( "-d", "--debug", nargs="?", type=int, default=0, metavar="LEVEL", help="Print debugging info. (Larger LEVEL means more info.)", ) parser.add_argument( "--fill", nargs="?", type=lambda s: unicode(s).lower(), choices=["no_fill", "fg_fill", "bg_fill"], default="no_fill", help="Select fill/no-fill for schematic symbol boxes.", ) args = parser.parse_args() # kipart f1.csv f2.csv # Create f1.lib, f2.lib # kipart f1.csv f2.csv -w # Overwrite f1.lib, f2.lib # kipart f1.csv f2.csv -a # Append to f1.lib, f2.lib # kipart f1.csv f2.csv -o f.lib # Create f.lib # kipart f1.csv f2.csv -w -o f.lib # Overwrite f.lib # kipart f1.csv f2.csv -a -o f.lib # Append to f.lib # Load the function for reading the part description file. part_reader_name = args.reader + "_reader" # Name of the reader module. reader_dir = readers[args.reader] sys.path.append(reader_dir) # Import from dir where the reader is if reader_dir == ".": importlib.import_module(part_reader_name) # Import module. reader_module = sys.modules[part_reader_name] # Get imported module. else: importlib.import_module("kipart." + part_reader_name) # Import module. reader_module = sys.modules[ "kipart." + part_reader_name ] # Get imported module. part_reader = getattr(reader_module, part_reader_name) # Get reader function. DEFAULT_PIN.side = args.side check_file_exists = True # Used to check for existence of a single output lib file. for input_file in args.input_files: # No explicit output lib file, so each individual input file will generate its own .lib file. if check_file_exists or not args.output: output_file = args.output or os.path.splitext(input_file)[0] + ".lib" if os.path.isfile(output_file): # The output lib file already exists. if args.overwrite: # Overwriting an existing file, so ignore the existing parts. parts_lib = OrderedDict() elif args.append: # Appending to an existing file, so read in existing parts. parts_lib = read_lib_file(output_file) else: print( "Output file {} already exists! Use the --overwrite option to replace it or the --append option to append to it.".format( output_file ) ) sys.exit(1) else: # Lib file doesn't exist, so create a new lib file starting with no parts. parts_lib = OrderedDict() # Don't setup the output lib file again if -o option was used to specify a single output lib. check_file_exists = not args.output file_ext = os.path.splitext(input_file)[-1].lower() # Get input file extension. if file_ext == ".zip": # Process the individual files inside a ZIP archive. with zipfile.ZipFile(input_file, "r") as zip_file: for zipped_file in zip_file.infolist(): zip_file_ext = os.path.splitext(zipped_file.filename)[-1] if zip_file_ext in [".csv", ".txt"]: # Only process CSV, TXT, Excel files in the archive. with zip_file.open(zipped_file, "r") as part_data_file: part_data_file = io.TextIOWrapper(part_data_file) call_kipart( args, part_reader, part_data_file, zipped_file.filename, zip_file_ext, parts_lib, ) elif zip_file_ext in [".xlsx"]: xlsx_data = zip_file.read(zipped_file) part_data_file = io.BytesIO(xlsx_data) call_kipart( args, part_reader, part_data_file, zipped_file.filename, zip_file_ext, parts_lib, ) else: # Skip unrecognized files. continue elif file_ext in [".csv", ".txt"]: # Process CSV and TXT files. with open(input_file, "r") as part_data_file: call_kipart( args, part_reader, part_data_file, input_file, file_ext, parts_lib ) elif file_ext in [".xlsx"]: # Process Excel files. with open(input_file, "rb") as part_data_file: call_kipart( args, part_reader, part_data_file, input_file, file_ext, parts_lib ) else: # Skip unrecognized files. continue if not args.output: # No global output lib file, so output a lib file for each input file. write_lib_file(parts_lib, output_file) if args.output: # Only a single lib output file was given, so write library to it after all # the input files were processed. write_lib_file(parts_lib, output_file)
import queue, threading, io, serial, pynmea2 from time import sleep import logging from pitop.miniscreen import Miniscreen from PIL import Image, ImageDraw, ImageFont gpsSerial = serial.Serial('/dev/ttyS0', 9600, timeout=1.) gpsIO = io.TextIOWrapper(io.BufferedRWPair(gpsSerial, gpsSerial)) gpsSerial.write(b'$PMTK314,0,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1*34\r\n') ms = Miniscreen() image = Image.new( ms.mode, ms.size, ) canvas = ImageDraw.Draw(image) ms.set_max_fps(60) nmea_list = ['GGA', 'GSA', 'RMC', 'ZDA'] nmea_dict = { 'latitude': "", 'longitude': "", 'speed': "", 'date_time': "", 'satelites': 0, 'altitude': "", 'fix': True, 'fix_type': "" } q = queue.Queue()
def readlines(): for line in io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8'): yield line.strip('\n')
class Robot: global x_p global c rospy.init_node('omoros', anonymous=True) # fetch /global parameters param_port = rospy.get_param('~port') param_baud = rospy.get_param('~baud') param_modelName = rospy.get_param('~modelName') # Open Serial port with parameter settings ser = serial.Serial(param_port, param_baud) #ser = serial.Serial('/dev/ttyS0', 115200) #For raspberryPi ser_io = io.TextIOWrapper(io.BufferedRWPair(ser, ser, 1), newline='\r', line_buffering=True) config = VehicleConfig() pose = MyPose() goal_1 = MyGoal_1() goal_2 = MyGoal_2() joyAxes = [] joyButtons = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] # Buttons 15 joyDeadband = 0.15 exp = 0.3 # Joystick expo setting isAutoMode = True isArrowMode = False # Whether to control robo with arrow key or not arrowCon = ArrowCon #initialize data cmd = Command enc_L = 0.0 # Left wheel encoder count from QENCOD message enc_R = 0.0 # Right wheel encoder count from QENCOD message enc_L_prev = 0.0 enc_R_prev = 0.0 enc_offset_L = 0.0 enc_offset_R = 0.0 enc_cnt = 0 odo_L = 0.0 # Left Wheel odometry returned from QODO message odo_R = 0.0 # Right Wheel odometry returned from QODO message RPM_L = 0.0 # Left Wheel RPM returned from QRPM message RPM_R = 0.0 # Right Wheel RPM returned from QRPM message speedL = 0.0 # Left Wheel speed returned from QDIFF message speedR = 0.0 # Reft Wheel speed returned from QDIFF message vel = 0.0 # Velocity returned from CVW command rot = 0.0 # Rotational speed returned from CVR command def __init__(self): ## Set vehicle specific configurations if self.param_modelName == "r1": print "**********" print "Driving R1" print "**********" self.config.WIDTH = 0.591 # Apply vehicle width for R1 version self.config.WHEEL_R = 0.11 # Apply wheel radius for R1 version self.config.WHEEL_MAXV = 1200.0 # Maximum speed can be applied to each wheel (mm/s) self.config.V_Limit = 0.6 # Limited speed (m/s) self.config.W_Limit = 0.1 self.config.V_Limit_JOY = 0.25 # Limited speed for joystick control self.config.W_Limit_JOY = 0.05 self.config.ArrowFwdStep = 250 # Steps move forward based on Odometry self.config.ArrowRotRate = 0.125 self.config.encoder.Dir = 1.0 self.config.encoder.PPR = 1000 self.config.encoder.GearRatio = 15 elif self.param_modelName == "mini": print "***************" print "Driving R1-mini" print "***************" self.config.WIDTH = 0.170 # Apply vehicle width for mini version self.config.WHEEL_R = 0.0336 # Apply wheel radius for mini version self.config.WHEEL_MAXV = 500.0 self.config.V_Limit = 0.5 self.config.W_Limit = 0.2 self.config.V_Limit_JOY = 0.5 self.config.W_Limit_JOY = 0.1 self.config.ArrowFwdStep = 100 self.config.ArrowRotRate = 0.1 self.config.encoder.Dir = -1.0 self.config.encoder.PPR = 234 self.config.encoder.GearRatio = 21 else: print "Error: param:modelName, Only support r1 and mini. exit..." exit() print('Wheel Track:{:.2f}m, Radius:{:.2f}m'.format( self.config.WIDTH, self.config.WHEEL_R)) self.config.BodyCircumference = self.config.WIDTH * math.pi print('Platform Rotation arc length: {:04f}m'.format( self.config.BodyCircumference)) self.config.WheelCircumference = self.config.WHEEL_R * 2 * math.pi print('Wheel circumference: {:04f}m'.format( self.config.WheelCircumference)) self.config.encoder.Step = self.config.WheelCircumference / ( self.config.encoder.PPR * self.config.encoder.GearRatio * 4) print('Encoder step: {:04f}m/pulse'.format(self.config.encoder.Step)) print(self.ser.name) # Print which port was really used self.joyAxes = [0, 0, 0, 0, 0, 0, 0, 0] self.joyButtons = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] # Configure data output if self.ser.isOpen(): print("Serial Open") self.resetODO() sleep(0.05) self.reset_odometry() self.setREGI(0, 'QENCOD') sleep(0.05) self.setREGI(1, 'QODO') sleep(0.05) self.setREGI(2, 'QDIFFV') sleep(0.05) self.setREGI(3, '0') sleep(0.05) self.setREGI(4, '0') #self.setREGI(3,'QVW') #sleep(0.05) #self.setREGI(4,'QRPM') sleep(0.05) self.setSPERI(20) sleep(0.05) self.setPEEN(1) sleep(0.05) self.reset_odometry() # Subscriber rospy.Subscriber("joy", Joy, self.callbackJoy) rospy.Subscriber('/scan', LaserScan, self.callback) rospy.Subscriber("cmd_vel", Twist, self.callbackCmdVel) # publisher self.pub_enc_l = rospy.Publisher('motor/encoder/left', Float64, queue_size=10) self.pub_enc_r = rospy.Publisher('motor/encoder/right', Float64, queue_size=10) self.pub_motor_status = rospy.Publisher('motor/status', R1MotorStatusLR, queue_size=10) self.odom_pub = rospy.Publisher("odom", Odometry, queue_size=10) self.odom_broadcaster = TransformBroadcaster() rate = rospy.Rate(rospy.get_param('~hz', 30)) # 30hz rospy.Timer(rospy.Duration(0.05), self.joytimer) rospy.Timer(rospy.Duration(0.01), self.serReader) #rospy.Timer(rospy.Duration(0.05), self.callback) self.pose.timestamp = rospy.Time.now() while not rospy.is_shutdown(): if self.cmd.isAlive == True: self.cmd.cnt += 1 if self.cmd.cnt > 1000: #Wait for about 3 seconds self.cmd.isAlive = False self.isAutoMode = False rate.sleep() self.ser.close() def callback(self, msg): global x_p s0 = 0 s1 = 0 s2 = 0 s2 = numpy.mean(msg.ranges[0:24]) s1 = numpy.mean(msg.ranges[108:132]) s0 = numpy.mean(msg.ranges[216:240]) #print('=========================================') #print(s0)#left #print(s1) #print(s2)#right #print(msg.ranges) if abs(x_p) < 1: if (s0 < 0.5) or (s1 < 0.5) or (s2 < 0.5): a = 0 b = 0 elif (s1 > 0.5): a = 50 b = 50 else: a = 0 b = 0 else: a = 0 b = 0 cmd = '$CDIFFV,{:.0f},{:.0f}'.format(a, b) self.ser.write(cmd + "\r" + "\n") def MoveGoal(self, pose, goal, msg): global x_p global c x_p = pose.x y_p = pose.y theta_p = pose.theta x_g = goal.x y_g = goal.y theta_g = goal.theta print(pose.x - goal.x) if (x_p != x_g): if abs(x_p) < abs(x_g / 2): a = 100 * x_p + 25 else: a = (x_g - x_p) * 100 + 10 if a > 250: a = 250 if a < -250: a = -250 a = a if -0.01 < (pose.x - goal.x) < 0.01: c = 0 cmd = '$CDIFFV,{:.0f},{:.0f}'.format(a, a) if self.ser.isOpen(): print cmd self.ser.write(cmd + "\r" + "\n") def serReader(self, event): global x_p global c reader = self.ser_io.readline() if reader: packet = reader.split(",") try: header = packet[0].split("#")[1] if header.startswith('QVW'): self.vel = int(packet[1]) self.rot = int(packet[2]) elif header.startswith('QENCOD'): enc_L = int(packet[1]) enc_R = int(packet[2]) if self.enc_cnt == 0: self.enc_offset_L = enc_L self.enc_offset_R = enc_R self.enc_cnt += 1 self.enc_L = enc_L * self.config.encoder.Dir - self.enc_offset_L self.enc_R = enc_R * self.config.encoder.Dir - self.enc_offset_R self.pub_enc_l.publish(Float64(data=self.enc_L)) self.pub_enc_r.publish(Float64(data=self.enc_R)) self.pose = self.updatePose(self.pose, self.enc_L, self.enc_R) #print('Encoder:L{:.2f}, R:{:.2f}'.format(self.enc_L, self.enc_R)) elif header.startswith('QODO'): self.odo_L = float(packet[1]) * self.config.encoder.Dir self.odo_R = float(packet[2]) * self.config.encoder.Dir #print('Odo:{:.2f}mm,{:.2f}mm'.format(self.odo_L, self.odo_R)) elif header.startswith('QRPM'): self.RPM_L = int(packet[1]) self.RPM_R = int(packet[2]) #print('RPM:{:.2f}mm,{:.2f}mm'.format(self.RPM_L, self.RPM_R)) elif header.startswith('QDIFFV'): self.speedL = int(packet[1]) self.speedR = int(packet[2]) except: pass status_left = R1MotorStatus(low_voltage=0, overloaded=0, power=0, encoder=self.enc_L, RPM=self.RPM_L, ODO=self.odo_L, speed=self.speedL) status_right = R1MotorStatus(low_voltage=0, overloaded=0, power=0, encoder=self.enc_R, RPM=self.RPM_R, ODO=self.odo_R, speed=self.speedR) self.pub_motor_status.publish( R1MotorStatusLR(header=Header(stamp=rospy.Time.now()), Vspeed=self.vel, Vomega=self.rot, left=status_left, right=status_right)) def callbackJoy(self, data): self.joyAxes = deepcopy(data.axes) #print('Joy:{:.2f},{:.2f}'.format(self.joyAxes[0], self.joyAxes[1])) # Read the most recent button state newJoyButtons = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] newJoyButtons = deepcopy(data.buttons) # Check if button 1(B) is newly set if (newJoyButtons[1] == 1) and (newJoyButtons[1] != self.joyButtons[1]): if self.isAutoMode != True: self.isAutoMode = True print "In Auto mode" else: self.isAutoMode = False print "In Manual mode" if (newJoyButtons[10] == 1) and (newJoyButtons[10] != self.joyButtons[10]): if self.isArrowMode != True: self.isArrowMode = True self.arrowCon.isFinished = True print "Joystick Arrow Mode" else: self.isArrowMode = False print "Joystick Axis Mode" if self.isArrowMode == True: #if (newJoyButtons[13]==1) or (newJoyButtons[14]==1): #if (self.joyAxes[7]==1.0) or (self.joyAxes[7]==-1.0): if (self.joyAxes[5] == 1.0) or (self.joyAxes[5] == -1.0): if self.arrowCon.isFinished == True: self.arrowCon.isFinished = False #if newJoyButtons[13]==1: # FWD arrow #if self.joyAxes[7]==1.0: if self.joyAxes[5] == 1.0: self.arrowCommand("FWD", self.arrowCon, self.config) else: self.arrowCommand("REAR", self.arrowCon, self.config) #print "Arrow: {:.2f} {:.2f} ".format(self.arrowCon.startOdo_L, self.arrowCon.targetOdo_L) #elif (newJoyButtons[11]==1) or (newJoyButtons[12]==1): #For Xbox360 controller #elif (self.joyAxes[6]==1.0) or (self.joyAxes[6]==-1.0): elif (self.joyAxes[4] == 1.0) or (self.joyAxes[4] == -1.0): if self.arrowCon.isFinished == True: turnRate = 10.5 self.arrowCon.isFinished = False #if newJoyButtons[11]==1: # Left arrow #if self.joyAxes[6]==1.0: if self.joyAxes[4] == 1.0: self.arrowCommand("LEFT", self.arrowCon, self.config) else: # Right arrow self.arrowCommand("RIGHT", self.arrowCon, self.config) # Update button state self.joyButtons = deepcopy(newJoyButtons) def arrowCommand(self, command, arrowCon, config): if command == "FWD": arrowCon.setFwd = 1 arrowCon.targetOdo_L = self.odo_L + config.ArrowFwdStep #target 1 step ahead arrowCon.targetOdo_R = self.odo_R + config.ArrowFwdStep #target 1 step ahead print "Arrow Fwd" elif command == "REAR": self.arrowCon.setFwd = -1 self.arrowCon.targetOdo_L = self.odo_L - self.config.ArrowFwdStep #target 1 step rear self.arrowCon.targetOdo_R = self.odo_R - self.config.ArrowFwdStep #target 1 step rear print "Arrow Rev" elif command == "LEFT": arrowCon.setRot = 1 arrowCon.targetOdo_L = self.odo_L - config.BodyCircumference * 1000 * config.ArrowRotRate arrowCon.targetOdo_R = self.odo_R + config.BodyCircumference * 1000 * config.ArrowRotRate print "Arrow Left" elif command == "RIGHT": arrowCon.setRot = -1 arrowCon.targetOdo_L = self.odo_L + config.BodyCircumference * 1000 * config.ArrowRotRate arrowCon.targetOdo_R = self.odo_R - config.BodyCircumference * 1000 * config.ArrowRotRate print "Arrow Right" def callbackCmdVel(self, cmd): """ Set wheel speed from cmd message from auto navigation """ if self.isAutoMode: #print "CMD_VEL: {:.2f} {:.2f} ".format(cmd.linear.x, cmd.angular.z) cmdV = cmd.linear.x cmdW = cmd.angular.z if cmdV > self.config.V_Limit: cmdV = self.config.V_Limit elif cmdV < -self.config.V_Limit: cmdV = -self.config.V_Limit if cmdW > self.config.W_Limit: cmdW = self.config.W_Limit elif cmdW < -self.config.W_Limit: cmdW = -self.config.W_Limit (speedL, speedR) = self.getWheelSpeedLR(self.config, cmdV, cmdW) #print "SPEED LR: {:.2f} {:.2f} ".format(speedL, speedR) self.sendCDIFFVcontrol(speedL * 200, speedR * 200) def reset_odometry(self): self.last_speedL = 0.0 self.last_speedR = 0.0 def joytimer(self, event): if not self.isAutoMode: self.joy_v = self.joyAxes[1] self.joy_w = self.joyAxes[0] #print "Joy mode: {:.2f} {:.2f} ".format(self.joy_v, self.joy_w) else: return if not self.isArrowMode: # Apply joystick deadband and calculate vehicle speed (mm/s) and rate of chage of orientation(rad/s) joyV = 0.0 joyR = 0.0 if abs(self.joy_v) < self.joyDeadband: joyV = 0.0 else: joyV = (1 - self.exp) * self.joy_v + ( self.exp) * self.joy_v * self.joy_v * self.joy_v if abs(self.joy_w) < self.joyDeadband: joyR = 0.0 else: joyR = (1 - self.exp) * self.joy_w + ( self.exp) * self.joy_w * self.joy_w * self.joy_w # Apply max Vehicle speed (speedL, speedR) = self.getWheelSpeedLR(self.config, joyV * self.config.V_Limit_JOY, joyR * self.config.W_Limit_JOY) #print "Joystick VL, VR: {:.2f} {:.2f}".format(speedL, speedR) self.sendCDIFFVcontrol(speedL * 1000, speedR * 1000) else: if self.arrowCon.isFinished == False: if self.arrowCon.setFwd == 1: # For forward motion if (self.odo_L < self.arrowCon.targetOdo_L) or ( self.odo_R < self.arrowCon.targetOdo_R): #print "Fwd: {:.2f} {:.2f} ".format(self.odo_L, self.odo_R) self.sendCDIFFVcontrol(100, 100) else: self.sendCDIFFVcontrol(0, 0) self.arrowCon.isFinished = True self.arrowCon.setFwd = 0 print "Finished!" elif self.arrowCon.setFwd == -1: if (self.odo_L > self.arrowCon.targetOdo_L) or ( self.odo_R > self.arrowCon.targetOdo_R): #print "Rev: {:.2f} {:.2f} ".format(self.odo_L, self.odo_R) self.sendCDIFFVcontrol(-100, -100) else: self.sendCDIFFVcontrol(0, 0) self.arrowCon.isFinished = True self.arrowCon.setFwd = 0 print "Finished!" elif self.arrowCon.setRot == 1: #Turning left if (self.odo_L > self.arrowCon.targetOdo_L) or ( self.odo_R < self.arrowCon.targetOdo_R): #print "Left: {:.2f} {:.2f} ".format(self.odo_L, self.odo_R) self.sendCDIFFVcontrol(-100, 100) else: self.sendCDIFFVcontrol(0, 0) self.arrowCon.isFinished = True self.arrowCon.setRot = 0 print "Finished!" elif self.arrowCon.setRot == -1: #Turning Right if (self.odo_L < self.arrowCon.targetOdo_L) or ( self.odo_R > self.arrowCon.targetOdo_R): #print "Right: {:.2f} {:.2f} ".format(self.odo_L, self.odo_R) self.sendCDIFFVcontrol(100, -100) else: self.sendCDIFFVcontrol(0, 0) self.arrowCon.isFinished = True self.arrowCon.setRot = 0 print "Finished!" def updatePose(self, pose, encoderL, encoderR): """Update Position x,y,theta from encoder count L, R Return new Position x,y,theta""" global x_p print c now = rospy.Time.now() dL = encoderL - self.enc_L_prev dR = encoderR - self.enc_R_prev self.enc_L_prev = encoderL self.enc_R_prev = encoderR dT = (now - pose.timestamp) / 1000000.0 pose.timestamp = now x = pose.x y = pose.y theta = pose.theta R = 0.0 if (dR - dL) == 0: R = 0.0 else: R = self.config.WIDTH / 2.0 * (dL + dR) / (dR - dL) Wdt = (dR - dL) * self.config.encoder.Step / self.config.WIDTH ICCx = x - R * np.sin(theta) ICCy = y + R * np.cos(theta) pose.x = np.cos(Wdt) * (x - ICCx) - np.sin(Wdt) * (y - ICCy) + ICCx pose.y = np.sin(Wdt) * (x - ICCx) + np.cos(Wdt) * (y - ICCy) + ICCy pose.theta = theta + Wdt twist = TwistWithCovariance() twist.twist.linear.x = self.vel / 1000.0 twist.twist.linear.y = 0.0 twist.twist.angular.z = self.rot / 1000.0 Vx = twist.twist.linear.x Vy = twist.twist.linear.y Vth = twist.twist.angular.z odom_quat = quaternion_from_euler(0, 0, pose.theta) self.odom_broadcaster.sendTransform((pose.x, pose.y, 0.), odom_quat, now, 'base_link', 'odom') #self.odom_broadcaster.sendTransform((pose.x,pose.y,0.),odom_quat,now,'base_footprint','odom') odom = Odometry() odom.header.stamp = now odom.header.frame_id = 'odom' odom.pose.pose = Pose(Point(pose.x, pose.y, 0.), Quaternion(*odom_quat)) odom.child_frame_id = 'base_link' #odom.child_frame_id = 'base_footprint' odom.twist.twist = Twist(Vector3(Vx, Vy, 0), Vector3(0, 0, Vth)) print "x:{:.2f} y:{:.2f} theta:{:.2f}".format( pose.x, pose.y, pose.theta * 180 / math.pi) self.odom_pub.publish(odom) x_p = pose.x return pose def getWheelSpeedLR(self, config, V_m_s, W_rad_s): """Takes Speed V (m/s) and Rotational sepeed W(rad/s) and compute each wheel speed in m/s Kinematics reference from http://enesbot.me/kinematic-model-of-a-differential-drive-robot.html""" speedL = V_m_s - config.WIDTH * W_rad_s / config.WHEEL_R / 2.0 speedR = V_m_s + config.WIDTH * W_rad_s / config.WHEEL_R / 2.0 return speedL, speedR def sendCVWcontrol(self, config, V_mm_s, W_mrad_s): """ Set Vehicle velocity and rotational speed """ if V_mm_s > config.V_Limit: V_mm_s = config.V_Limit elif V_mm_s < -config.V_Limit: V_mm_s = -config.V_limit if W_mrad_s > config.W_Limit: W_mrad_s = config.W_Limit elif W_mrad_s < -config.W_Limit: W_mrad_s = -config.W_Limit # Make a serial message to be sent to motor driver unit cmd = '$CVW,{:.0f},{:.0f}'.format(V_mm_s, W_mrad_s) print cmd if self.ser.isOpen(): print cmd self.ser.write(cmd + "\r" + "\n") def sendCDIFFVcontrol(self, VLmm_s, VRmm_s): """ Set differential wheel speed for Left and Right """ if VLmm_s > self.config.WHEEL_MAXV: VLmm_s = self.config.WHEEL_MAXV elif VLmm_s < -self.config.WHEEL_MAXV: VLmm_s = -self.config.WHEEL_MAXV if VRmm_s > self.config.WHEEL_MAXV: VRmm_s = self.config.WHEEL_MAXV elif VRmm_s < -self.config.WHEEL_MAXV: VRmm_s = -self.config.WHEEL_MAXV # Make a serial message to be sent to motor driver unit cmd = '$CDIFFV,{:.0f},{:.0f}'.format(VLmm_s, VRmm_s) if self.ser.isOpen(): #print cmd self.ser.write(cmd + "\r" + "\n") def setREGI(self, param1, param2): msg = "$SREGI," + str(param1) + ',' + param2 self.ser.write(msg + "\r" + "\n") def setSPERI(self, param): msg = "$SPERI," + str(param) self.ser.write(msg + "\r" + "\n") def setPEEN(self, param): msg = "$SPEEN," + str(param) self.ser.write(msg + "\r" + "\n") def resetODO(self): self.ser.write("$SODO\r\n")
def main( argsl=None, # type: List[str] args=None, # type: argparse.Namespace job_order_object=None, # type: MutableMapping[Text, Any] stdin=sys.stdin, # type: IO[Any] stdout=None, # type: Union[TextIO, StreamWriter] stderr=sys.stderr, # type: IO[Any] versionfunc=versionstring, # type: Callable[[], Text] logger_handler=None, # custom_schema_callback=None, # type: Callable[[], None] executor=None, # type: Callable[..., Tuple[Dict[Text, Any], Text]] loadingContext=None, # type: LoadingContext runtimeContext=None # type: RuntimeContext ): # type: (...) -> int if not stdout: # force UTF-8 even if the console is configured differently if (hasattr(sys.stdout, "encoding") # type: ignore and sys.stdout.encoding != 'UTF-8'): # type: ignore if PY3 and hasattr(sys.stdout, "detach"): stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') else: stdout = getwriter('utf-8')(sys.stdout) # type: ignore else: stdout = cast(TextIO, sys.stdout) # type: ignore _logger.removeHandler(defaultStreamHandler) if logger_handler is not None: stderr_handler = logger_handler else: stderr_handler = logging.StreamHandler(stderr) _logger.addHandler(stderr_handler) # pre-declared for finally block workflowobj = None prov_log_handler = None # type: Optional[logging.StreamHandler] try: if args is None: if argsl is None: argsl = sys.argv[1:] args = arg_parser().parse_args(argsl) if args.record_container_id: if not args.cidfile_dir: args.cidfile_dir = os.getcwd() del args.record_container_id if runtimeContext is None: runtimeContext = RuntimeContext(vars(args)) else: runtimeContext = runtimeContext.copy() # If on Windows platform, a default Docker Container is used if not # explicitely provided by user if onWindows() and not runtimeContext.default_container: # This docker image is a minimal alpine image with bash installed # (size 6 mb). source: https://github.com/frol/docker-alpine-bash runtimeContext.default_container = windows_default_container_id # If caller parsed its own arguments, it may not include every # cwltool option, so fill in defaults to avoid crashing when # dereferencing them in args. for key, val in iteritems(get_default_args()): if not hasattr(args, key): setattr(args, key, val) # Configure logging rdflib_logger = logging.getLogger("rdflib.term") rdflib_logger.addHandler(stderr_handler) rdflib_logger.setLevel(logging.ERROR) if args.quiet: # Silence STDERR, not an eventual provenance log file stderr_handler.setLevel(logging.WARN) if runtimeContext.debug: # Increase to debug for both stderr and provenance log file _logger.setLevel(logging.DEBUG) rdflib_logger.setLevel(logging.DEBUG) formatter = None # type: Optional[logging.Formatter] if args.timestamps: formatter = logging.Formatter("[%(asctime)s] %(message)s", "%Y-%m-%d %H:%M:%S") stderr_handler.setFormatter(formatter) ## if args.version: print(versionfunc()) return 0 _logger.info(versionfunc()) if args.print_supported_versions: print("\n".join(supported_cwl_versions(args.enable_dev))) return 0 if not args.workflow: if os.path.isfile("CWLFile"): setattr(args, "workflow", "CWLFile") else: _logger.error("") _logger.error( "CWL document required, no input file was provided") arg_parser().print_help() return 1 if args.relax_path_checks: command_line_tool.ACCEPTLIST_RE = command_line_tool.ACCEPTLIST_EN_RELAXED_RE if args.ga4gh_tool_registries: ga4gh_tool_registries[:] = args.ga4gh_tool_registries if not args.enable_ga4gh_tool_registry: del ga4gh_tool_registries[:] if custom_schema_callback is not None: custom_schema_callback() elif args.enable_ext: res = pkg_resources.resource_stream(__name__, 'extensions.yml') use_custom_schema("v1.0", "http://commonwl.org/cwltool", res.read()) res.close() else: use_standard_schema("v1.0") if args.provenance: if not args.compute_checksum: _logger.error( "--provenance incompatible with --no-compute-checksum") return 1 ro = ResearchObject(temp_prefix_ro=args.tmpdir_prefix, orcid=args.orcid, full_name=args.cwl_full_name) runtimeContext.research_obj = ro log_file_io = ro.open_log_file_for_activity(ro.engine_uuid) prov_log_handler = logging.StreamHandler(log_file_io) class ProvLogFormatter(logging.Formatter): """Enforce ISO8601 with both T and Z.""" def __init__(self): # type: () -> None super(ProvLogFormatter, self).__init__("[%(asctime)sZ] %(message)s") def formatTime(self, record, datefmt=None): # type: (logging.LogRecord, str) -> str record_time = time.gmtime(record.created) formatted_time = time.strftime("%Y-%m-%dT%H:%M:%S", record_time) with_msecs = "%s,%03d" % (formatted_time, record.msecs) return with_msecs prov_log_handler.setFormatter(ProvLogFormatter()) _logger.addHandler(prov_log_handler) _logger.debug(u"[provenance] Logging to %s", log_file_io) if argsl is not None: # Log cwltool command line options to provenance file _logger.info("[cwltool] %s %s", sys.argv[0], u" ".join(argsl)) _logger.debug(u"[cwltool] Arguments: %s", args) if loadingContext is None: loadingContext = LoadingContext(vars(args)) else: loadingContext = loadingContext.copy() loadingContext.loader = default_loader( loadingContext.fetcher_constructor) loadingContext.research_obj = runtimeContext.research_obj loadingContext.disable_js_validation = \ args.disable_js_validation or (not args.do_validate) loadingContext.construct_tool_object = getdefault( loadingContext.construct_tool_object, workflow.default_make_tool) loadingContext.resolver = getdefault(loadingContext.resolver, tool_resolver) loadingContext.do_update = not (args.pack or args.print_subgraph) uri, tool_file_uri = resolve_tool_uri( args.workflow, resolver=loadingContext.resolver, fetcher_constructor=loadingContext.fetcher_constructor) try_again_msg = "" if args.debug else ", try again with --debug for more information" try: job_order_object, input_basedir, jobloader = load_job_order( args, stdin, loadingContext.fetcher_constructor, loadingContext.overrides_list, tool_file_uri) if args.overrides: loadingContext.overrides_list.extend( load_overrides(file_uri(os.path.abspath(args.overrides)), tool_file_uri)) loadingContext, workflowobj, uri = fetch_document( uri, loadingContext) assert loadingContext.loader is not None if args.print_deps: printdeps(workflowobj, loadingContext.loader, stdout, args.relative_deps, uri) return 0 loadingContext, uri \ = resolve_and_validate_document(loadingContext, workflowobj, uri, preprocess_only=(args.print_pre or args.pack), skip_schemas=args.skip_schemas) assert loadingContext.loader is not None processobj, metadata = loadingContext.loader.resolve_ref(uri) processobj = cast(CommentedMap, processobj) if args.pack: stdout.write( print_pack(loadingContext.loader, processobj, uri, metadata)) return 0 if args.provenance and runtimeContext.research_obj: # Can't really be combined with args.pack at same time runtimeContext.research_obj.packed_workflow( print_pack(loadingContext.loader, processobj, uri, metadata)) if args.print_pre: stdout.write( json_dumps(processobj, indent=4, sort_keys=True, separators=(',', ': '))) return 0 tool = make_tool(uri, loadingContext) if args.make_template: def my_represent_none(self, data): # pylint: disable=unused-argument """Force clean representation of 'null'.""" return self.represent_scalar(u'tag:yaml.org,2002:null', u'null') yaml.RoundTripRepresenter.add_representer( type(None), my_represent_none) yaml.round_trip_dump(generate_input_template(tool), sys.stdout, default_flow_style=False, indent=4, block_seq_indent=2) return 0 if args.validate: print("{} is valid CWL.".format(args.workflow)) return 0 if args.print_rdf: stdout.write( printrdf(tool, loadingContext.loader.ctx, args.rdf_serializer)) return 0 if args.print_dot: printdot(tool, loadingContext.loader.ctx, stdout) return 0 if args.print_targets: for f in ("outputs", "steps", "inputs"): if tool.tool[f]: _logger.info("%s%s targets:", f[0].upper(), f[1:-1]) stdout.write(" " + "\n ".join( [shortname(t["id"]) for t in tool.tool[f]]) + "\n") return 0 if args.target: if isinstance(tool, Workflow): url = urllib.parse.urlparse(tool.tool["id"]) if url.fragment: extracted = get_subgraph( [tool.tool["id"] + "/" + r for r in args.target], tool) else: extracted = get_subgraph([ loadingContext.loader.fetcher.urljoin( tool.tool["id"], "#" + r) for r in args.target ], tool) else: _logger.error("Can only use --target on Workflows") return 1 loadingContext.loader.idx[extracted["id"]] = extracted tool = make_tool(extracted["id"], loadingContext) if args.print_subgraph: if "name" in tool.tool: del tool.tool["name"] stdout.write( json_dumps(tool.tool, indent=4, sort_keys=True, separators=(',', ': '))) return 0 except (validate.ValidationException) as exc: _logger.error(u"Tool definition failed validation:\n%s", exc, exc_info=args.debug) return 1 except (RuntimeError, WorkflowException) as exc: _logger.error(u"Tool definition failed initialization:\n%s", exc, exc_info=args.debug) return 1 except Exception as exc: _logger.error( u"I'm sorry, I couldn't load this CWL file%s.\nThe error was: %s", try_again_msg, exc if not args.debug else "", exc_info=args.debug) return 1 if isinstance(tool, int): return tool # If on MacOS platform, TMPDIR must be set to be under one of the # shared volumes in Docker for Mac # More info: https://dockstore.org/docs/faq if sys.platform == "darwin": default_mac_path = "/private/tmp/docker_tmp" if runtimeContext.tmp_outdir_prefix == DEFAULT_TMP_PREFIX: runtimeContext.tmp_outdir_prefix = default_mac_path if runtimeContext.tmpdir_prefix == DEFAULT_TMP_PREFIX: runtimeContext.tmpdir_prefix = default_mac_path for dirprefix in ("tmpdir_prefix", "tmp_outdir_prefix", "cachedir"): if getattr(runtimeContext, dirprefix) and getattr( runtimeContext, dirprefix) != DEFAULT_TMP_PREFIX: sl = "/" if getattr(runtimeContext, dirprefix).endswith("/") or dirprefix == "cachedir" \ else "" setattr( runtimeContext, dirprefix, os.path.abspath(getattr(runtimeContext, dirprefix)) + sl) if not os.path.exists( os.path.dirname(getattr(runtimeContext, dirprefix))): try: os.makedirs( os.path.dirname(getattr(runtimeContext, dirprefix))) except Exception as e: _logger.error("Failed to create directory: %s", e) return 1 if args.cachedir: if args.move_outputs == "move": runtimeContext.move_outputs = "copy" runtimeContext.tmp_outdir_prefix = args.cachedir runtimeContext.secret_store = getdefault(runtimeContext.secret_store, SecretStore()) runtimeContext.make_fs_access = getdefault( runtimeContext.make_fs_access, StdFsAccess) try: initialized_job_order_object = init_job_order( job_order_object, args, tool, jobloader, stdout, print_input_deps=args.print_input_deps, relative_deps=args.relative_deps, make_fs_access=runtimeContext.make_fs_access, input_basedir=input_basedir, secret_store=runtimeContext.secret_store) except SystemExit as err: return err.code if not executor: if args.parallel: executor = MultithreadedJobExecutor() runtimeContext.select_resources = executor.select_resources else: executor = SingleJobExecutor() assert executor is not None try: runtimeContext.basedir = input_basedir del args.workflow del args.job_order conf_file = getattr(args, "beta_dependency_resolvers_configuration", None) # Text use_conda_dependencies = getattr(args, "beta_conda_dependencies", None) # Text if conf_file or use_conda_dependencies: runtimeContext.job_script_provider = DependenciesConfiguration( args) else: runtimeContext.find_default_container = functools.partial( find_default_container, default_container=runtimeContext.default_container, use_biocontainers=args.beta_use_biocontainers) (out, status) = executor(tool, initialized_job_order_object, runtimeContext, logger=_logger) if out is not None: if runtimeContext.research_obj is not None: runtimeContext.research_obj.create_job(out, None, True) def loc_to_path(obj): for field in ("path", "nameext", "nameroot", "dirname"): if field in obj: del obj[field] if obj["location"].startswith("file://"): obj["path"] = uri_file_path(obj["location"]) visit_class(out, ("File", "Directory"), loc_to_path) # Unsetting the Generation from final output object visit_class(out, ("File", ), MutationManager().unset_generation) if isinstance(out, string_types): stdout.write(out) else: stdout.write( json_dumps( out, indent=4, # type: ignore ensure_ascii=False)) stdout.write("\n") if hasattr(stdout, "flush"): stdout.flush() # type: ignore if status != "success": _logger.warning(u"Final process status is %s", status) return 1 _logger.info(u"Final process status is %s", status) return 0 except (validate.ValidationException) as exc: _logger.error(u"Input object failed validation:\n%s", exc, exc_info=args.debug) return 1 except UnsupportedRequirement as exc: _logger.error(u"Workflow or tool uses unsupported feature:\n%s", exc, exc_info=args.debug) return 33 except WorkflowException as exc: _logger.error(u"Workflow error%s:\n%s", try_again_msg, strip_dup_lineno(Text(exc)), exc_info=args.debug) return 1 except Exception as exc: # pylint: disable=broad-except _logger.error(u"Unhandled error%s:\n %s", try_again_msg, exc, exc_info=args.debug) return 1 finally: if args and runtimeContext and runtimeContext.research_obj \ and workflowobj: research_obj = runtimeContext.research_obj assert loadingContext is not None assert loadingContext.loader is not None prov_dependencies = prov_deps(workflowobj, loadingContext.loader, uri) research_obj.generate_snapshot(prov_dependencies) if prov_log_handler is not None: # Stop logging so we won't half-log adding ourself to RO _logger.debug(u"[provenance] Closing provenance log file %s", prov_log_handler) _logger.removeHandler(prov_log_handler) # Ensure last log lines are written out prov_log_handler.flush() # Underlying WritableBagFile will add the tagfile to the manifest prov_log_handler.stream.close() prov_log_handler.close() research_obj.close(args.provenance) _logger.removeHandler(stderr_handler) _logger.addHandler(defaultStreamHandler)
def p(*c): print(*c) # _*_ coding:utf-8 _*_ import pandas as pd import numpy as np import io import sys sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') def loadDataSet(): return [[1, 3, 4], [2, 3, 5], [1, 2, 3, 5], [2, 5]] def createC1(dataSet): C1 = [] for transaction in dataSet: for item in transaction: if not [item] in C1: C1.append([item]) # C1.sort() return list(map(frozenset, C1))#use frozen set so we #can use it as a key in a dict def scanD(D, Ck, minSupport): ssCnt = {} for tid in D: for can in Ck: if can.issubset(tid): if can not in ssCnt: ssCnt[can]=1 else:
# -*- coding: utf-8 -*- import re import json import base64 import binascii import rsa import requests import io # for test import sys sys.path.insert(0, '..') sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='gb18030') from config import WBCLIENT, user_agent from config import USER_NAME, PASSWD from logger import logger session = requests.session() session.headers['User-Agent'] = user_agent def encrypt_passwd(passwd, pubkey, servertime, nonce): #rsa加密密码 key = rsa.PublicKey(int(pubkey, 16), int( '10001', 16)) #int('10001', 16) 并非要转换成16进制,而是说10001是16 进制的数,int函数要将其转化为十进制 message = str(servertime) + '\t' + str(nonce) + '\n' + str(passwd) passwd = rsa.encrypt(message.encode('utf-8'), key) return binascii.b2a_hex(passwd) # 登录微博的流程
def split_wet_file(wet_file_path, counter_inc_fn=None): """Split a WET file into separate pages.""" logging.info("Splitting file: %s", wet_file_path) if not counter_inc_fn: counter_inc_fn = get_counter_inc_fn("split-wet-file") counter_inc_fn("wet-file") with tf.io.gfile.GFile(wet_file_path, "rb") as f, gzip.GzipFile( fileobj=f) as g: url = None content = None content_len = None content_type = None timestamp = None def _maybe_get_page(): """Generate a (url, {features}) page.""" if not url and url is not None: counter_inc_fn("page-filitered-nourl") if not content and content is not None: counter_inc_fn("page-filtered-nocontent") if not content_type and content_type is not None: counter_inc_fn("page-nocontenttype") if not content_len and content_len is not None: counter_inc_fn("page-nocontentlen") if not timestamp and timestamp is not None: counter_inc_fn("page-notimestamp") if content and url: counter_inc_fn("page-emitted") return (url, { "text": "\n".join(content), "content-type": content_type, "content-length": content_len, "timestamp": timestamp, "url": url }) return None for line in io.TextIOWrapper(g, encoding="utf-8"): line = line.strip() if not line: continue if line == _PAGE_DELIMITER: page = _maybe_get_page() if page: yield page url = "" content = [] content_len = "" content_type = "" timestamp = "" if line.startswith(_URL_KEY): url = line[len(_URL_KEY):].strip() if line.startswith(_URL_DATE): timestamp = line[len(_URL_DATE):].strip() if line.startswith(_CONTENT_TYPE): content_type = line[len(_CONTENT_TYPE):].strip() if line.startswith(_CONTENT_LEN): content_len = line[len(_CONTENT_LEN):].strip() if line.startswith(_METADATA_PREFIXES): continue content.append(line) page = _maybe_get_page() if page: yield page
def _set_output(self, outfiles: OutputFiles): self._filters = [] self._outfiles = outfiles filter_wrapper = self._filter_wrapper() for filter_class, outfile in ( (RestFileWriter, outfiles.rest), (InfoFileWriter, outfiles.info), (WildcardFileWriter, outfiles.wildcard), ): if outfile: textiowrapper = io.TextIOWrapper(outfile) self._textiowrappers.append(textiowrapper) self._filters.append(filter_wrapper(None, filter_class(textiowrapper), None)) # minimum length and maximum length for lengths, file1, file2, filter_class in ( (self._minimum_length, outfiles.too_short, outfiles.too_short2, TooShortReadFilter), (self._maximum_length, outfiles.too_long, outfiles.too_long2, TooLongReadFilter) ): if lengths is None: continue writer = self._open_writer(file1, file2) if file1 else None f1 = filter_class(lengths[0]) if lengths[0] is not None else None if len(lengths) == 2 and lengths[1] is not None: f2 = filter_class(lengths[1]) else: f2 = None self._filters.append(filter_wrapper(writer, filter=f1, filter2=f2)) if self.max_n is not None: f1 = f2 = NContentFilter(self.max_n) self._filters.append(filter_wrapper(None, f1, f2)) if self.discard_casava: f1 = f2 = CasavaFilter() self._filters.append(filter_wrapper(None, f1, f2)) if int(self.discard_trimmed) + int(self.discard_untrimmed) + int(outfiles.untrimmed is not None) > 1: raise ValueError('discard_trimmed, discard_untrimmed and outfiles.untrimmed must not ' 'be set simultaneously') if outfiles.demultiplex: self._demultiplexer = self._create_demultiplexer(outfiles) self._filters.append(self._demultiplexer) else: # Allow overriding the wrapper for --discard-untrimmed/--untrimmed-(paired-)output untrimmed_filter_wrapper = self._untrimmed_filter_wrapper() # Set up the remaining filters to deal with --discard-trimmed, # --discard-untrimmed and --untrimmed-output. These options # are mutually exclusive in order to avoid brain damage. if self.discard_trimmed: self._filters.append( filter_wrapper(None, DiscardTrimmedFilter(), DiscardTrimmedFilter())) elif self.discard_untrimmed: self._filters.append( untrimmed_filter_wrapper(None, DiscardUntrimmedFilter(), DiscardUntrimmedFilter())) elif outfiles.untrimmed: untrimmed_writer = self._open_writer(outfiles.untrimmed, outfiles.untrimmed2) self._filters.append( untrimmed_filter_wrapper(untrimmed_writer, DiscardUntrimmedFilter(), DiscardUntrimmedFilter())) self._filters.append(self._final_filter(outfiles))
def create(): fa = _make_argument_optional(fs.open_output_stream, metadata=None)(path) fp = FileProxy(fa, path, lambda: fs.open_output_stream(path)) return io.TextIOWrapper(fa, encoding=encoding)
def reader(buffer, title=None): print(f'{title}: ', ' '.join((io.TextIOWrapper(buffer.stderr, encoding='utf-8')))) return
import gzip, io, csv import numpy as np import matplotlib.pyplot as plt from sklearn.neural_network import MLPClassifier from sklearn.externals import joblib from sklearn.decomposition import PCA from sklearn import metrics X, y = [], [] with io.TextIOWrapper(gzip.open("../data/train.csv.gz", "r")) as csvfile: csvreader = csv.reader(csvfile, delimiter=',', quotechar='|') for i, row in enumerate(csvreader): if i == 0: continue vals = row[1:] X.append(vals) y.append([row[0]]) X = np.array(X, dtype=float) X = X / 255 y = np.array(y, dtype=str) # plot the first 10 images def gimg(i): # images must be 28x28x3 return np.reshape( # greyscale images using the same value for R G B np.column_stack((X[i], X[i], X[i])), (28, 28, 3))
def isolation(self, input=None, env=None, color=False): """A context manager that sets up the isolation for invoking of a command line tool. This sets up stdin with the given input data and `os.environ` with the overrides from the given dictionary. This also rebinds some internals in Click to be mocked (like the prompt functionality). This is automatically done in the :meth:`invoke` method. .. versionadded:: 4.0 The ``color`` parameter was added. :param input: the input stream to put into sys.stdin. :param env: the environment overrides as dictionary. :param color: whether the output should contain color codes. The application can still override this explicitly. """ input = make_input_stream(input, self.charset) old_stdin = sys.stdin old_stdout = sys.stdout old_stderr = sys.stderr old_forced_width = formatting.FORCED_WIDTH formatting.FORCED_WIDTH = 80 env = self.make_env(env) if PY2: bytes_output = StringIO() if self.echo_stdin: input = EchoingStdin(input, bytes_output) sys.stdout = bytes_output if not self.mix_stderr: bytes_error = StringIO() sys.stderr = bytes_error else: bytes_output = io.BytesIO() if self.echo_stdin: input = EchoingStdin(input, bytes_output) input = io.TextIOWrapper(input, encoding=self.charset) sys.stdout = io.TextIOWrapper(bytes_output, encoding=self.charset) if not self.mix_stderr: bytes_error = io.BytesIO() sys.stderr = io.TextIOWrapper(bytes_error, encoding=self.charset) if self.mix_stderr: sys.stderr = sys.stdout sys.stdin = input def visible_input(prompt=None): sys.stdout.write(prompt or "") val = input.readline().rstrip("\r\n") sys.stdout.write("{}\n".format(val)) sys.stdout.flush() return val def hidden_input(prompt=None): sys.stdout.write("{}\n".format(prompt or "")) sys.stdout.flush() return input.readline().rstrip("\r\n") def _getchar(echo): char = sys.stdin.read(1) if echo: sys.stdout.write(char) sys.stdout.flush() return char default_color = color def should_strip_ansi(stream=None, color=None): if color is None: return not default_color return not color old_visible_prompt_func = termui.visible_prompt_func old_hidden_prompt_func = termui.hidden_prompt_func old__getchar_func = termui._getchar old_should_strip_ansi = utils.should_strip_ansi termui.visible_prompt_func = visible_input termui.hidden_prompt_func = hidden_input termui._getchar = _getchar utils.should_strip_ansi = should_strip_ansi old_env = {} try: for key, value in iteritems(env): old_env[key] = os.environ.get(key) if value is None: try: del os.environ[key] except Exception: pass else: os.environ[key] = value yield (bytes_output, not self.mix_stderr and bytes_error) finally: for key, value in iteritems(old_env): if value is None: try: del os.environ[key] except Exception: pass else: os.environ[key] = value sys.stdout = old_stdout sys.stderr = old_stderr sys.stdin = old_stdin termui.visible_prompt_func = old_visible_prompt_func termui.hidden_prompt_func = old_hidden_prompt_func termui._getchar = old__getchar_func utils.should_strip_ansi = old_should_strip_ansi formatting.FORCED_WIDTH = old_forced_width
def setup_locale(locale, localization_proxy=None, text_mode=False): """ Procedure setting the system to use the given locale and store it in to the localization module (if given). DOES NOT PERFORM ANY CHECKS OF THE GIVEN LOCALE. $LANG must be set by the caller in order to set the language used by gettext. Doing this in a thread-safe way is up to the caller. We also try to set a proper console font for the locale in text mode. If the font for the locale can't be displayed in the Linux console, we fall back to the English locale. This function returns the locale that was used in the setlocale call, which, depending on what the environment and interface is able to support, may be different from the locale requested. :param str locale: locale to setup :param localization_proxy: DBus proxy of the localization module or None :param bool text_mode: if the locale is being setup for text mode :return: the locale that was actually set :rtype: str """ if localization_proxy: localization_proxy.SetLanguage(locale) # not all locales might be displayable in text mode if text_mode: # check if the script corresponding to the locale/language # can be displayed by the Linux console # * all scripts for the given locale/language need to be # supported by the linux console # * otherwise users might get a screen full of white rectangles # (also known as "tofu") in text mode # then we also need to check if we have information about what # font to use for correctly displaying the given language/locale script_supported = locale_supported_in_console(locale) log.debug("scripts found for locale %s: %s", locale, get_locale_scripts(locale)) console_fonts = get_locale_console_fonts(locale) log.debug("console fonts found for locale %s: %s", locale, console_fonts) font_set = False if script_supported and console_fonts: # try to set console font for font in console_fonts: if set_console_font(font): # console font set successfully, skip the rest font_set = True break if not font_set: log.warning("can't set console font for locale %s", locale) # report what exactly went wrong if not (script_supported): log.warning("script not supported by console for locale %s", locale) if not (console_fonts): # no fonts known for locale log.warning("no console font found for locale %s", locale) if script_supported and console_fonts: log.warning( "none of the suggested fonts can be set for locale %s", locale) log.warning("falling back to the English locale") locale = constants.DEFAULT_LANG os.environ["LANG"] = locale # pylint: disable=environment-modify # set the locale to the value we have selected # Since glibc does not install all locales, an installable locale may not # actually be available right now. Give it a shot and fallback. log.debug("setting locale to: %s", locale) setenv("LANG", locale) try: locale_mod.setlocale(locale_mod.LC_ALL, locale) except locale_mod.Error as e: log.debug("setlocale failed: %s", e) locale = constants.DEFAULT_LANG setenv("LANG", locale) locale_mod.setlocale(locale_mod.LC_ALL, locale) # This part is pretty gross: # In python3, sys.stdout and friends are TextIOWrapper objects that translate # betwen str types (in python) and byte types (used to actually read from or # write to the console). These wrappers are configured at startup using the # locale at startup, which means that if anaconda starts with LANG=C or LANG # unset, sys.stdout.encoding will be "ascii". And if the language is then changed, # because anaconda read the kickstart or parsed the command line or whatever, # say, to Czech, text mode will crash because it's going to try to print non-ASCII # characters and sys.stdout doesn't know what to do with them. So, when changing # the locale, create new objects for the standard streams so they are created with # the new locale's encoding. # Replacing stdout is about as stable as it looks, and it seems to break when done # after the GUI is started. Only make the switch if the encoding actually changed. if locale_mod.getpreferredencoding() != sys.stdout.encoding: sys.stdin = io.TextIOWrapper(sys.stdin.detach()) sys.stdout = io.TextIOWrapper(sys.stdout.detach()) sys.stderr = io.TextIOWrapper(sys.stderr.detach()) return locale
def open(self, index, subindex=0, mode="rb", encoding="ascii", buffering=1024, size=None, block_transfer=False, force_segment=False, request_crc_support=True): """Open the data stream as a file like object. :param int index: Index of object to open. :param int subindex: Sub-index of object to open. :param str mode: ========= ========================================================== Character Meaning --------- ---------------------------------------------------------- 'r' open for reading (default) 'w' open for writing 'b' binary mode (default) 't' text mode ========= ========================================================== :param str encoding: The str name of the encoding used to decode or encode the file. This will only be used in text mode. :param int buffering: An optional integer used to set the buffering policy. Pass 0 to switch buffering off (only allowed in binary mode), 1 to select line buffering (only usable in text mode), and an integer > 1 to indicate the size in bytes of a fixed-size chunk buffer. :param int size: Size of data to that will be transmitted. :param bool block_transfer: If block transfer should be used. :param bool force_segment: Force use of segmented download regardless of data size. :param bool request_crc_support: If crc calculation should be requested when using block transfer :returns: A file like object. """ buffer_size = buffering if buffering > 1 else io.DEFAULT_BUFFER_SIZE if "r" in mode: if block_transfer: raw_stream = BlockUploadStream(self, index, subindex, request_crc_support=request_crc_support) else: raw_stream = ReadableStream(self, index, subindex) if buffering: buffered_stream = io.BufferedReader(raw_stream, buffer_size=buffer_size) else: return raw_stream if "w" in mode: if block_transfer: raw_stream = BlockDownloadStream(self, index, subindex, size, request_crc_support=request_crc_support) else: raw_stream = WritableStream(self, index, subindex, size, force_segment) if buffering: buffered_stream = io.BufferedWriter(raw_stream, buffer_size=buffer_size) else: return raw_stream if "b" not in mode: # Text mode line_buffering = buffering == 1 return io.TextIOWrapper(buffered_stream, encoding, line_buffering=line_buffering) return buffered_stream
def main(): """The entry point for the program.""" if '--' in sys.argv: cli_args = sys.argv[1:sys.argv.index('--')] generator_args = sys.argv[sys.argv.index('--') + 1:] else: cli_args = sys.argv[1:] generator_args = [] args = _cmdline_parser.parse_args(cli_args) debug = False if args.verbose is None: logging_level = logging.WARNING elif args.verbose == 1: logging_level = logging.INFO elif args.verbose == 2: logging_level = logging.DEBUG debug = True else: print('error: I can only be so garrulous, try -vv.', file=sys.stderr) sys.exit(1) logging.basicConfig(level=logging_level) if args.spec and args.spec[0].startswith('+') and args.spec[0].endswith( '.py'): # Hack: Special case for defining a spec in Python for testing purposes # Use this if you want to define a Stone spec using a Python module. # The module should should contain an api variable that references a # :class:`stone.api.Api` object. try: api = imp.load_source('api', args.api[0]).api except ImportError as e: print('error: Could not import API description due to:', e, file=sys.stderr) sys.exit(1) else: if args.spec: specs = [] read_from_stdin = False for spec_path in args.spec: if spec_path == '-': read_from_stdin = True elif not spec_path.endswith('.stone'): print( "error: Specification '%s' must have a .stone extension." % spec_path, file=sys.stderr) sys.exit(1) elif not os.path.exists(spec_path): print("error: Specification '%s' cannot be found." % spec_path, file=sys.stderr) sys.exit(1) else: with open(spec_path) as f: specs.append((spec_path, f.read())) if read_from_stdin and specs: print( "error: Do not specify stdin and specification files " "simultaneously.", file=sys.stderr) sys.exit(1) if not args.spec or read_from_stdin: specs = [] if debug: print('Reading specification from stdin.') if six.PY2: UTF8Reader = codecs.getreader('utf8') sys.stdin = UTF8Reader(sys.stdin) stdin_text = sys.stdin.read() else: stdin_text = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8').read() parts = stdin_text.split('namespace') if len(parts) == 1: specs.append(('stdin.1', parts[0])) else: specs.append(('stdin.1', '%snamespace%s' % (parts.pop(0), parts.pop(0)))) while parts: specs.append(('stdin.%s' % (len(specs) + 1), 'namespace%s' % parts.pop(0))) if args.filter_by_route_attr: route_filter, route_filter_errors = parse_route_attr_filter( args.filter_by_route_attr, debug) if route_filter_errors: print('Error(s) in route filter:', file=sys.stderr) for err in route_filter_errors: print(err, file=sys.stderr) sys.exit(1) else: route_filter = None # TODO: Needs version tower = TowerOfStone(specs, debug=debug) try: api = tower.parse() except InvalidSpec as e: print('%s:%s: error: %s' % (e.path, e.lineno, e.msg), file=sys.stderr) if debug: print( 'A traceback is included below in case this is a bug in ' 'Stone.\n', traceback.format_exc(), file=sys.stderr) sys.exit(1) if api is None: print( 'You must fix the above parsing errors for generation to ' 'continue.', file=sys.stderr) sys.exit(1) if args.whitelist_namespace_routes: for namespace_name in args.whitelist_namespace_routes: if namespace_name not in api.namespaces: print( 'error: Whitelisted namespace missing from spec: %s' % namespace_name, file=sys.stderr) sys.exit(1) for namespace in api.namespaces.values(): if namespace.name not in args.whitelist_namespace_routes: namespace.routes = [] namespace.route_by_name = {} if args.blacklist_namespace_routes: for namespace_name in args.blacklist_namespace_routes: if namespace_name not in api.namespaces: print( 'error: Blacklisted namespace missing from spec: %s' % namespace_name, file=sys.stderr) sys.exit(1) else: api.namespaces[namespace_name].routes = [] api.namespaces[namespace_name].route_by_name = {} if route_filter: for namespace in api.namespaces.values(): filtered_routes = [] for route in namespace.routes: if route_filter.eval(route): filtered_routes.append(route) else: del namespace.route_by_name[route.name] namespace.routes = filtered_routes if args.attribute: attrs = set(args.attribute) if ':all' in attrs: attrs = {field.name for field in api.route_schema.fields} else: attrs = set() for namespace in api.namespaces.values(): for route in namespace.routes: for k in list(route.attrs.keys()): if k not in attrs: del route.attrs[k] # Remove attrs that weren't specified from the route schema for field in api.route_schema.fields[:]: if field.name not in attrs: api.route_schema.fields.remove(field) del api.route_schema._fields_by_name[field.name] else: attrs.remove(field.name) # Error if specified attr isn't even a field in the route schema if attrs: attr = attrs.pop() print('error: Attribute not defined in stone_cfg.Route: %s' % attr, file=sys.stderr) sys.exit(1) if args.generator in _builtin_generators: generator_module = __import__('stone.target.%s' % args.generator, fromlist=['']) elif not os.path.exists(args.generator): print("error: Generator '%s' cannot be found." % args.generator, file=sys.stderr) sys.exit(1) elif not os.path.isfile(args.generator): print("error: Generator '%s' must be a file." % args.generator, file=sys.stderr) sys.exit(1) elif not Compiler.is_stone_generator(args.generator): print("error: Generator '%s' must have a .stoneg.py extension." % args.generator, file=sys.stderr) sys.exit(1) else: # A bit hacky, but we add the folder that the generator is in to our # python path to support the case where the generator imports other # files in its local directory. new_python_path = os.path.dirname(args.generator) if new_python_path not in sys.path: sys.path.append(new_python_path) try: generator_module = imp.load_source('user_generator', args.generator) except: print( "error: Importing generator '%s' module raised an exception:" % args.generator, file=sys.stderr) raise c = Compiler( api, generator_module, generator_args, args.output, clean_build=args.clean_build, ) try: c.build() except GeneratorException as e: print('%s: error: %s raised an exception:\n%s' % (args.generator, e.generator_name, e.traceback), file=sys.stderr) sys.exit(1) if not sys.argv[0].endswith('stone'): # If we aren't running from an entry_point, then return api to make it # easier to do debugging. return api
#============================================================================= # Globals # Python has different string types depending on the python version. We must # be able to identify them both. # pylint: disable=undefined-variable STRTYPES = [str] if sys.version_info > (3, ) else [str, unicode] # Pandoc uses UTF-8 for both input and output; so must its filters. This is # handled differently depending on the python version. if sys.version_info > (3, ): # Py3 strings are unicode: https://docs.python.org/3.5/howto/unicode.html. # Character encoding/decoding is performed automatically at stream # interfaces: https://stackoverflow.com/questions/16549332/. # Set it to UTF-8 for all streams. STDIN = io.TextIOWrapper(sys.stdin.buffer, 'utf-8', 'strict') STDOUT = io.TextIOWrapper(sys.stdout.buffer, 'utf-8', 'strict') STDERR = io.TextIOWrapper(sys.stderr.buffer, 'utf-8', 'strict') else: # Py2 strings are ASCII bytes. Encoding/decoding is handled separately. # See: https://docs.python.org/2/howto/unicode.html. STDIN = sys.stdin STDOUT = sys.stdout STDERR = sys.stdout # Privately flags that cleveref TeX needs to be written into the doc # (for TeX/pdf output only). _cleveref_tex_flag = False # pylint: disable=invalid-name # Used to track section numbers # pylint: disable=invalid-name