def download_and_repack(country_code=None,
network_code=None,
circle=None,
token=None,
source=None,
destination=None,
byte_order="b",
verbose=False):
"""
Downloads and packs the base station data.
So far, only opencellid.org supported.
Args:
country_code (int): the country code;
network_code (int): the network code;
circle (tuple): latitude, longitude (degrees) and radius in km;
token (str): service token;
source (str): downloaded file name;
destination (str): destination file name;
mnc_block_size (int): the size of the mnc block;
mcc_block_size (int): the size of the mcc block;
byte_order (str): byte order;
verbose (bool): prints verbose output;
"""
def v(*args, **kwargs):
if verbose:
print(*args, **kwargs)
byte_order = byte_order.lower()
if byte_order not in "bl":
raise ValueError("Unknown byte order: {}".format(byte_order))
byte_order = dict(b=">", l="<")[byte_order]
if destination is None:
if country_code is None:
destination = "all.bin"
else:
if network_code is None:
destination = "{country_code}.bin".format(
country_code=country_code)
else:
destination = "{country_code}-{network_code}.bin".format(
country_code=country_code, network_code=network_code)
v("Target: {}".format(destination))
if source is None:
if token is None:
v("No token specified: downloading from git")
if country_code is None:
raise ValueError("Cannot download worldwide database yet")
else:
url = "https://github.com/pulkin/agps-data/raw/master/opencellid.org/{country_code}.csv.gz".format(
country_code=country_code)
else:
if country_code is None:
url = "https://opencellid.org/ocid/downloads?token={token}&type=full&file=cell_towers.csv.gz".format(
token=token)
else:
url = "https://opencellid.org/ocid/downloads?token={token}&type=mcc&file={country_code}.csv.gz".format(
token=token, country_code=country_code)
v("Downloading {} ...".format(url))
response = urllib.request.urlopen(
urllib.request.Request(
url,
headers={
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
}))
buf = io.BytesIO(response.read())
else:
v("Reading {} ...".format(source))
buf = open(source, 'rb')
v("Unzipping ...")
buf_raw = gzip.GzipFile(fileobj=buf, mode='rb')
buf_txt = io.TextIOWrapper(buf_raw)
v("Parsing ...")
dtype = [
('radio_type', 'S4'),
('mcc', byte_order + 'u2'),
('mnc', byte_order + 'u2'),
('area_code', byte_order + 'u2'),
('cell', byte_order + 'u2'),
('lon', byte_order + 'f4'),
('lat', byte_order + 'f4'),
]
data = numpy.genfromtxt(buf_txt,
dtype=dtype,
skip_header=1,
delimiter=",",
usecols=(0, 1, 2, 3, 4, 6, 7))
v("Filtering ...")
data = data[data["radio_type"] == b"GSM"][[
"mcc", "mnc", "area_code", "cell", "lon", "lat"
]]
if country_code is not None:
v(" - mcc: {:d}".format(len(data)), end="")
data = data[data["mcc"] == country_code]
v(" -> {:d}".format(len(data)))
if network_code is not None:
v(" - mnc: {:d}".format(len(data)), end="")
data = data[data["mnc"] == network_code]
v(" -> {:d}".format(len(data)))
if circle is not None:
v(" - circle: {:d}".format(len(data)), end="")
theta0, phi0, r0 = circle
theta = (data["lat"] - theta0) * numpy.pi / 180
phi = (data["lon"] - phi0) * numpy.pi / 180
mask = (theta**2 + phi**2 * numpy.cos(theta0)**2) < (r0 * 1e3 /
earth_radius)**2
data = data[mask]
v(" -> {:d}".format(len(data)))
if len(data) == 0:
raise ValueError("No data to save")
v("Sorting ...")
data = numpy.sort(data, order=("mcc", "mnc", "area_code", "cell"))
v("Items total: {:d}".format(len(data)))
v("Preparing tables ...")
keys = "mcc", "mnc"
mask = numpy.zeros(len(data), dtype=bool)
mask[0] = True
for k in keys:
mask[1:] |= data[k][1:] != data[k][:-1]
table_ptrs = numpy.where(mask)[0]
table_data = recfunctions.repack_fields(data[table_ptrs][list(keys)])
v("Saving ...")
with open(destination, 'wb') as f:
f.write(b'agps-bin')
f.write(b'\x00')
f.write({">": b">", "<": b"<"}[byte_order])
f.write(struct.pack(byte_order + "L", len(table_ptrs)))
for _d, _p in zip(table_data, table_ptrs):
f.write(struct.pack(byte_order + "HHL", *_d, _p))
recfunctions.repack_fields(data[["area_code", "cell", "lon",
"lat"]]).tofile(f)
v("Total size: {:d} bytes".format(f.tell()))
v("Done")
segment.strip() for split in splits[:-1]
for segment in [split, glossary] if segment != ''
]
return segments + [splits[-1].strip()
] if splits[-1] != '' else segments
if __name__ == '__main__':
# python 2/3 compatibility
if sys.version_info < (3, 0):
sys.stderr = codecs.getwriter('UTF-8')(sys.stderr)
sys.stdout = codecs.getwriter('UTF-8')(sys.stdout)
sys.stdin = codecs.getreader('UTF-8')(sys.stdin)
else:
sys.stdin = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
sys.stdout = io.TextIOWrapper(sys.stdout.buffer,
encoding='utf-8',
write_through=True,
line_buffering=True)
parser = create_parser()
args = parser.parse_args()
# read/write files as UTF-8
args.codes = codecs.open(args.codes.name, encoding='utf-8')
if args.input.name != '<stdin>':
args.input = codecs.open(args.input.name, encoding='utf-8')
if args.output.name != '<stdout>':
args.output = codecs.open(args.output.name, 'w', encoding='utf-8')
import urllib.request
from urllib.parse import urlparse
import sys
import io
sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding = 'utf-8')
sys.stderr = io.TextIOWrapper(sys.stderr.detach(), encoding = 'utf-8')
url = "http://www.encar.com/"
mem = urllib.request.urlopen(url)
print(type(mem))
print("geturl :",mem.geturl()) #결과 : http://www.encar.com/index.do
print("status :",mem.status) #응답코드 200 (정상), 404(요청페이지 없음), 403 (Reject), 500 (서버 에러)
print("headers :",mem.getheaders())
print("info :",mem.info(),"\n") #== headers
print("getcode :",mem.getcode()) #== status
print("read :",mem.read(10).decode('utf-8')) #해당 숫자만큼 불러옴
print(urlparse('http://www.encar.co.kr?test=test').query)
def create():
fa = fs.open_input_file(path)
fp = FileProxy(fa, path, lambda: fs.open_input_file(path))
return io.TextIOWrapper(fp, encoding=encoding)
def _json_decode(self, json_bytes, encoding):
tiow = io.TextIOWrapper(io.BytesIO(json_bytes), encoding=encoding, newline="")
obj = json.load(tiow)
tiow.close()
return obj
def write_data_export_zip(self, request, response):
with zipfile.ZipFile(response, mode="w") as zip_file:
with zip_file.open(f"{self.slug}.json",
mode="w") as json_binary_file, io.TextIOWrapper(
json_binary_file) as json_file:
json.dump(
{
"email":
self.user.email,
"date_joined":
self.user.date_joined.isoformat(),
"last_login":
self.user.last_login.isoformat(),
"url":
request.build_absolute_uri(self.get_absolute_url()),
"is_public":
self.is_public,
"is_approved":
self.is_approved,
"name":
self.name,
"city_or_town":
self.city_or_town,
"country":
self.country,
"cause_areas":
list(map(CauseArea.label, self.cause_areas)),
"cause_areas_other":
self.cause_areas_other,
"available_to_volunteer":
self.available_to_volunteer,
"open_to_job_offers":
self.open_to_job_offers,
"expertise_areas":
list(map(ExpertiseArea.label, self.expertise_areas)),
"expertise_areas_other":
self.expertise_areas_other,
"career_interest_areas":
list(
map(ExpertiseArea.label,
self.career_interest_areas)),
"available_as_speaker":
self.available_as_speaker,
"topics_i_speak_about":
self.topics_i_speak_about,
"organisational_affiliations":
list(
map(
OrganisationalAffiliation.label,
self.organisational_affiliations,
)),
"summary":
self.summary,
"giving_pledges":
list(map(GivingPledge.label, self.giving_pledges)),
"member_of_local_groups": [
request.build_absolute_uri(
local_group.get_absolute_uri())
for local_group in self.local_groups.all()
],
"organiser_of_local_groups": [
request.build_absolute_uri(
local_group.get_absolute_uri())
for local_group in self.user.localgroup_set.all()
],
"aliases": [
request.build_absolute_uri(
urls.reverse("profile",
kwargs={"slug": slug.slug}))
for slug in self.slugs.filter(redirect=True)
],
"legacy_hub_url":
(self.legacy_record and request.build_absolute_uri(
urls.reverse(
"profile_legacy",
kwargs={"legacy_record": self.legacy_record},
))),
},
json_file,
indent=2,
)
if self.image:
with self.image.open() as image_src_file, zip_file.open(
self.slug + pathlib.PurePath(self.image.name).suffix,
mode="w") as image_dst_file:
shutil.copyfileobj(image_src_file, image_dst_file)
def __init__(self,
executable,
configuration,
model,
analysis_threads=10,
search_threads=1,
output=False):
self.output = output
if output:
print(' Launching KataGo...')
def read_stream(name, stream, type_, buffer):
if output:
print(f' {name} thread has begun.')
while True:
line = stream.readline().rstrip()
if (not self._ready and type_ is LineType.error
and line.endswith(
'Started, ready to begin handling requests')):
self._ready = True
if output:
print(' KataGo is ready to accept inputs.')
if output:
print(f' {name} read: {line}')
if line:
buffer.append((type_, line))
else:
if output:
print(f' {name} thread has finished.')
break
command = f'{executable} analysis -config {configuration} -model {model} -analysis-threads {analysis_threads} ' \
f'-override-config numSearchThreads={search_threads}'
self._process = subprocess.Popen(command,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
self._buffer = []
self._ready = False
stderr = io.TextIOWrapper(self._process.stderr,
encoding='utf-8',
errors='strict')
self._errors = Thread(target=read_stream,
args=('ERR', stderr, LineType.error,
self._buffer))
self._errors.daemon = True
self._errors.start()
stdout = io.TextIOWrapper(self._process.stdout,
encoding='utf-8',
errors='strict')
self._outputs = Thread(target=read_stream,
args=('OUT', stdout, LineType.output,
self._buffer))
self._outputs.daemon = True
self._outputs.start()
if output:
print(' KataGo has launched.')
def textblock(filename,
start,
end,
compression=None,
encoding=system_encoding,
linesep=os.linesep,
buffersize=4096):
"""Pull out a block of text from a file given start and stop bytes.
This gets data starting/ending from the next linesep delimiter. Each block
consists of bytes in the range [start,end[, i.e. the stop byte is excluded.
If `start` is 0, then `start` corresponds to the true start byte. If
`start` is greater than 0 and does not point to the beginning of a new
line, then `start` is incremented until it corresponds to the start byte of
the next line. If `end` does not point to the beginning of a new line, then
the line that begins before `end` is included in the block although its
last byte exceeds `end`.
Examples
--------
>> with open('myfile.txt', 'wb') as f:
.. f.write('123\n456\n789\nabc')
In the example below, 1 and 10 don't line up with endlines.
>> u''.join(textblock('myfile.txt', 1, 10))
'456\n789\n'
"""
# Make sure `linesep` is not a byte string because
# `io.TextIOWrapper` in Python versions other than 2.7 dislike byte
# strings for the `newline` argument.
linesep = str(linesep)
# Get byte representation of the line separator.
bin_linesep = get_bin_linesep(encoding, linesep)
bin_linesep_len = len(bin_linesep)
if buffersize < bin_linesep_len:
error = ('`buffersize` ({0:d}) must be at least as large as the '
'number of line separator bytes ({1:d}).')
raise ValueError(error.format(buffersize, bin_linesep_len))
chunksize = end - start
with open(filename, 'rb', compression) as f:
with io.BufferedReader(f) as fb:
# If `start` does not correspond to the beginning of the file, we
# need to move the file pointer to `start - len(bin_linesep)`,
# search for the position of the next a line separator, and set
# `start` to the position after that line separator.
if start > 0:
# `start` is decremented by `len(bin_linesep)` to detect the
# case where the original `start` value corresponds to the
# beginning of a line.
start = max(0, start - bin_linesep_len)
# Set the file pointer to `start`.
fb.seek(start)
# Number of bytes to shift the file pointer before reading a
# new chunk to make sure that a multi-byte line separator, that
# is split by the chunk reader, is still detected.
shift = 1 - bin_linesep_len
while True:
buf = f.read(buffersize)
if len(buf) < bin_linesep_len:
raise StopIteration
try:
# Find the position of the next line separator and add
# `len(bin_linesep)` which yields the position of the
# first byte of the next line.
start += buf.index(bin_linesep)
start += bin_linesep_len
except ValueError:
# No line separator was found in the current chunk.
# Before reading the next chunk, we move the file
# pointer back `len(bin_linesep) - 1` bytes to make
# sure that a multi-byte line separator, that may have
# been split by the chunk reader, is still detected.
start += len(buf)
start += shift
fb.seek(shift, os.SEEK_CUR)
else:
# We have found the next line separator, so we need to
# set the file pointer to the first byte of the next
# line.
fb.seek(start)
break
with io.TextIOWrapper(fb, encoding, newline=linesep) as fbw:
# Retrieve and yield lines until the file pointer reaches
# `end`.
while start < end:
line = next(fbw)
# We need to encode the line again to get the byte length
# in order to correctly update `start`.
bin_line_len = len(line.encode(encoding))
if chunksize < bin_line_len:
error = ('`chunksize` ({0:d}) is less than the line '
'length ({1:d}). This may cause duplicate '
'processing of this line. It is advised to '
'increase `chunksize`.')
raise IOError(error.format(chunksize, bin_line_len))
yield line
start += bin_line_len
def main(argv=None):
if argv is None:
argv = sys.argv
if len(argv) < 3:
sys.exit('Usage: ' + argv[0] +
' <results-xml> [<witness-xml>]* [--no-overwrite-status].\n')
resultFile = argv[1]
witnessFiles = []
isOverwrite = True
for i in range(2, len(argv)):
if len(argv) > i and not argv[i].startswith('--'):
witnessFiles.append(argv[i])
if argv[i] == '--no-overwrite-status':
isOverwrite = False
if not os.path.exists(resultFile) or not os.path.isfile(resultFile):
sys.exit('File {0} does not exist.'.format(repr(resultFile)))
resultXML = tablegenerator.parse_results_file(resultFile)
witnessSets = []
for witnessFile in witnessFiles:
if not os.path.exists(witnessFile) or not os.path.isfile(witnessFile):
sys.exit('File {0} does not exist.'.format(repr(witnessFile)))
witnessXML = tablegenerator.parse_results_file(witnessFile)
witnessSets.append(getWitnesses(witnessXML))
resultXML.set(
'options', '' + resultXML.get('options', default='') + ' [[ ' +
witnessXML.get('options', default='') + ' ]]')
resultXML.set(
'date', '' + resultXML.get('date', default='') + ' [[ ' +
witnessXML.get('date', default='') + ' ]]')
for result in resultXML.findall('run'):
run = result.get('name')
basename = os.path.basename(run)
if 'correct' == result.findall('column[@title="category"]')[0].get(
'value'):
statusVer = result.findall('column[@title="status"]')[0]
categoryVer = result.findall('column[@title="category"]')[0]
properties = result.get('properties').split(' ')
expected_result = benchexec.result.satisfies_file_property(
basename, properties)
statusWit, categoryWit = (None, None)
i = 0
for witnessSet in witnessSets:
i = i + 1
witness = witnessSet.get(run, None)
# copy data from witness
if witness is not None:
for column in witness:
newColumn = ET.Element(
'column', {
'title':
'wit' + str(i) + '_' + column.get('title'),
'value': column.get('value'),
'hidden': column.get('hidden', 'false')
})
result.append(newColumn)
witnessSet.pop(run)
statusWitNew, categoryWitNew = getWitnessResult(
witness, expected_result)
if expected_result == False:
if statusWitNew.startswith(
'false(') or statusWit is None:
statusWit, categoryWit = (statusWitNew,
categoryWitNew)
if expected_result == True:
if statusWitNew.startswith(
'true') or statusWit is None:
statusWit, categoryWit = (statusWitNew,
categoryWitNew)
# Overwrite status with status from witness
if isOverwrite and statusWit is not None and categoryWit is not None:
result.findall('column[@title="status"]')[0].set(
'value', statusWit)
result.findall('column[@title="category"]')[0].set(
'value', categoryWit)
# Clean-up an entry that can be inferred by table-generator automatically, avoids path confusion
del result.attrib['logfile']
filename = resultFile + '.merged.xml.bz2'
print(' ' + filename)
open_func = bz2.BZ2File if hasattr(bz2.BZ2File,
'writable') else util.BZ2FileHack
with io.TextIOWrapper(open_func(filename, 'wb'),
encoding='utf-8') as xml_file:
xml_file.write(
xml_to_string(resultXML).replace(' \n', '').replace(' \n', ''))
#!/usr/bin/python
import gzip
import io
################################################################################# SRR3879604
# set dictionary and counters to empty or 0
bc_dic604 = {}
total_bc_count604 = 0
dic_count604 = 0
total_repeat_count604 = 0
no_bc_count604 = 0
# open with gzip, make sure to use 'rt'
with io.TextIOWrapper(
io.BufferedReader(
gzip.open(
'/projectnb/bf528/project_4_scrnaseq/fastq/SRR3879604/SRR3879604_1_bc.fastq.gz'
))) as sample604:
# loop through each line, gather barcodes, count barcode frequency, store count for non-bc samples too
for line in sample604:
if line[0] == '@':
next_line = next(sample604)
barcode = next_line[0:19]
if barcode in bc_dic604:
total_bc_count604 += 1
total_repeat_count604 += 1
bc_dic604[barcode] += 1
if barcode not in bc_dic604:
total_bc_count604 += 1
dic_count604 += 1
bc_dic604[barcode] = 1
else:
def autoconstruct(self):
# If camera device set in config use it, otherwise autodetect
cameradev = None
devicepaths = glob.glob("/dev/video*")
if self.config.args.camera_device:
self.logger.debug('camera_device specified: {}'.format(self.config.args.camera_device))
cameradev = self.config.args.camera_device
else:
# device not set, carry on and try to autodetect
for devicepath in sorted(devicepaths):
if not cameradev and self.check_input(devicepath):
cameradev = devicepath
self.logger.info('v4l2 device '+devicepath+' is a camera, autoselecting')
elif not cameradev:
self.logger.debug('v4l2 device '+devicepath+' is not a camera, ignoring')
if not cameradev:
raise ValueError('Error detecting camera video device')
# Check the camera has a valid input
try:
self.vd = io.TextIOWrapper(open(cameradev, "r+b", buffering=0))
cp = v4l2.v4l2_capability()
except Exception as e:
raise ValueError("Camera not specified in config, or camera not valid: {}".format(repr(e)))
if not self.check_input():
raise ValueError('Specified camera not valid')
# Log info
self.camera_info()
# Try and autodetect Jetson/Tegra CSI connection
if self.driver == 'tegra-video':
self.logger.info('Nvidia Jetson/Tegra CSI connection detected, switching to nvarguscamerasrc')
self.input = "nvarguscamerasrc"
elif 'input' not in self.config.args or not self.config.args.input:
self.input = "v4l2src"
else:
self.input = self.config.args.input
# Try and autodetect MFC device
self.mfcdev = None
for devicepath in devicepaths:
dp = io.TextIOWrapper(open(devicepath, "r+b", buffering=0))
ioctl(dp, v4l2.VIDIOC_QUERYCAP, cp)
if cp.card == "s5p-mfc-enc":
self.mfcdev = dp
self.logger.info(f'MFC Hardware encoder detected, autoselecting {devicepath}')
# If format set in config use it, otherwise autodetect
streamtype = None
if self.config.args.format:
streamtype = self.config.args.format
else:
if self.input == "nvarguscamerasrc":
self.logger.info('Nvidia Jetson/Tegra input detected, forcing Tegra stream format')
streamtype = 'tegra'
elif re.search("C920", self.card):
self.logger.info("Logitech C920 detected, forcing H264 passthrough")
streamtype = 'h264'
# format not set, carry on and try to autodetect
elif self.check_format('yuv'):
self.logger.info('Camera YUV stream available, using yuv stream')
streamtype = 'yuv'
# Otherwise, check for an mjpeg->h264 encoder pipeline.
elif self.check_format('mjpeg'):
self.logger.info('Camera MJPEG stream available, using mjpeg stream')
streamtype = 'mjpeg'
# Lastly look for a h264 stream
elif self.check_format('h264'):
self.logger.info('Camera H264 stream available, using H264 stream')
streamtype = 'h264'
if not streamtype:
raise ValueError('Error detecting camera video format')
# If encoder set in config use it, otherwise set to h264
encoder = None
if self.config.args.encoder:
encoder = self.config.args.encoder
if not encoder:
encoder = "h264"
self.logger.debug("Using encoder: {}".format(encoder))
# If raspberry camera detected set pixelformat to I420, otherwise set to YUY2 by default
pixelformat = "YUY2"
ioctl(self.vd, v4l2.VIDIOC_QUERYCAP, cp)
if cp.driver == "bm2835 mmal":
self.logger.info("Raspberry Pi Camera detected, setting pixel format to I420")
pixelformat = "I420"
# If raw pixelformat set in config override the defaults
if 'pixelformat' in self.config.args and self.config.args.pixelformat:
pixelformat = self.config.args.pixelformat
self.logger.debug("Using pixelformat: {}".format(pixelformat))
# Create and start the stream
try:
self.logger.info("Creating stream object - device: {}, stream: {}, pixelformat: {}, encoder: {}, input: {}".format(cameradev, streamtype, pixelformat, encoder, self.input))
Streamer(self.config, streamtype, pixelformat, encoder, self.input, cameradev)
if self.zeroconf:
# Update the stream advertisement with the new info
self.zeroconf.update({"stream":"replace_with_stream_info"})
except Exception as e:
if self.zeroconf:
self.zeroconf.update({"stream":""})
raise ValueError('Error creating {} stream: {}'.format(streamtype, repr(e)))
# Inform systemd that start is complete
self.logger.info("Notifying systemd of startup completion")
self.notify.notify("READY=1")
self.notify.notify("STATUS=Automatic Pipeline Initialisation Complete")
while not self._should_shutdown:
time.sleep(1)
def __iter__(self):
wrapper = io.TextIOWrapper(self.file)
yield from wrapper
def test_gzip_reader_textiowrapper(gzip_reader):
with gzip_reader("tests/file.txt.gz", "rb") as f:
wrapped = io.TextIOWrapper(f)
assert wrapped.read() == CONTENT
def main():
# Get Python routines for reading part description/CSV files.
readers = scan_for_readers()
parser = ap.ArgumentParser(
description="Generate single & multi-unit schematic symbols for KiCad from a CSV file."
)
parser.add_argument(
"-v", "--version", action="version", version="KiPart " + __version__
)
parser.add_argument(
"input_files",
nargs="+",
type=str,
metavar="file.[csv|txt|xlsx|zip]",
help="Files for parts in CSV/text/Excel format or as such files in .zip archives.",
)
parser.add_argument(
"-r",
"--reader",
nargs="?",
type=lambda s: unicode(s).lower(),
choices=readers.keys(),
default="generic",
help="Name of function for reading the CSV or part description files.",
)
parser.add_argument(
"-s",
"--sort",
nargs="?",
# type=str.lower,
type=lambda s: unicode(s).lower(),
choices=["row", "num", "name"],
default="row",
help="Sort the part pins by their entry order in the CSV file, their pin number, or their pin name.",
)
parser.add_argument(
"--reverse", action="store_true", help="Sort pins in reverse order."
)
parser.add_argument(
"--side",
nargs="?",
# type=str.lower,
type=lambda s: unicode(s).lower(),
choices=["left", "right", "top", "bottom"],
default="left",
help="Which side to place the pins by default.",
)
parser.add_argument(
"-o",
"--output",
nargs="?",
type=str,
metavar="file.lib",
help="Generated KiCad symbol library for parts.",
)
parser.add_argument(
"-f",
"--fuzzy_match",
action="store_true",
help="Use approximate string matching when looking-up the pin type, style and orientation.",
)
parser.add_argument(
"-b",
"--bundle",
action="store_true",
help="Bundle multiple, identically-named power and ground pins each into a single schematic pin.",
)
parser.add_argument(
"-a",
"--append",
"--add",
action="store_true",
help="Add parts to an existing part library. Overwrite existing parts only if used in conjunction with -w.",
)
parser.add_argument(
"-w",
"--overwrite",
action="store_true",
help="Allow overwriting of an existing part library.",
)
parser.add_argument(
"-d",
"--debug",
nargs="?",
type=int,
default=0,
metavar="LEVEL",
help="Print debugging info. (Larger LEVEL means more info.)",
)
parser.add_argument(
"--fill",
nargs="?",
type=lambda s: unicode(s).lower(),
choices=["no_fill", "fg_fill", "bg_fill"],
default="no_fill",
help="Select fill/no-fill for schematic symbol boxes.",
)
args = parser.parse_args()
# kipart f1.csv f2.csv # Create f1.lib, f2.lib
# kipart f1.csv f2.csv -w # Overwrite f1.lib, f2.lib
# kipart f1.csv f2.csv -a # Append to f1.lib, f2.lib
# kipart f1.csv f2.csv -o f.lib # Create f.lib
# kipart f1.csv f2.csv -w -o f.lib # Overwrite f.lib
# kipart f1.csv f2.csv -a -o f.lib # Append to f.lib
# Load the function for reading the part description file.
part_reader_name = args.reader + "_reader" # Name of the reader module.
reader_dir = readers[args.reader]
sys.path.append(reader_dir) # Import from dir where the reader is
if reader_dir == ".":
importlib.import_module(part_reader_name) # Import module.
reader_module = sys.modules[part_reader_name] # Get imported module.
else:
importlib.import_module("kipart." + part_reader_name) # Import module.
reader_module = sys.modules[
"kipart." + part_reader_name
] # Get imported module.
part_reader = getattr(reader_module, part_reader_name) # Get reader function.
DEFAULT_PIN.side = args.side
check_file_exists = True # Used to check for existence of a single output lib file.
for input_file in args.input_files:
# No explicit output lib file, so each individual input file will generate its own .lib file.
if check_file_exists or not args.output:
output_file = args.output or os.path.splitext(input_file)[0] + ".lib"
if os.path.isfile(output_file):
# The output lib file already exists.
if args.overwrite:
# Overwriting an existing file, so ignore the existing parts.
parts_lib = OrderedDict()
elif args.append:
# Appending to an existing file, so read in existing parts.
parts_lib = read_lib_file(output_file)
else:
print(
"Output file {} already exists! Use the --overwrite option to replace it or the --append option to append to it.".format(
output_file
)
)
sys.exit(1)
else:
# Lib file doesn't exist, so create a new lib file starting with no parts.
parts_lib = OrderedDict()
# Don't setup the output lib file again if -o option was used to specify a single output lib.
check_file_exists = not args.output
file_ext = os.path.splitext(input_file)[-1].lower() # Get input file extension.
if file_ext == ".zip":
# Process the individual files inside a ZIP archive.
with zipfile.ZipFile(input_file, "r") as zip_file:
for zipped_file in zip_file.infolist():
zip_file_ext = os.path.splitext(zipped_file.filename)[-1]
if zip_file_ext in [".csv", ".txt"]:
# Only process CSV, TXT, Excel files in the archive.
with zip_file.open(zipped_file, "r") as part_data_file:
part_data_file = io.TextIOWrapper(part_data_file)
call_kipart(
args,
part_reader,
part_data_file,
zipped_file.filename,
zip_file_ext,
parts_lib,
)
elif zip_file_ext in [".xlsx"]:
xlsx_data = zip_file.read(zipped_file)
part_data_file = io.BytesIO(xlsx_data)
call_kipart(
args,
part_reader,
part_data_file,
zipped_file.filename,
zip_file_ext,
parts_lib,
)
else:
# Skip unrecognized files.
continue
elif file_ext in [".csv", ".txt"]:
# Process CSV and TXT files.
with open(input_file, "r") as part_data_file:
call_kipart(
args, part_reader, part_data_file, input_file, file_ext, parts_lib
)
elif file_ext in [".xlsx"]:
# Process Excel files.
with open(input_file, "rb") as part_data_file:
call_kipart(
args, part_reader, part_data_file, input_file, file_ext, parts_lib
)
else:
# Skip unrecognized files.
continue
if not args.output:
# No global output lib file, so output a lib file for each input file.
write_lib_file(parts_lib, output_file)
if args.output:
# Only a single lib output file was given, so write library to it after all
# the input files were processed.
write_lib_file(parts_lib, output_file)
import queue, threading, io, serial, pynmea2
from time import sleep
import logging
from pitop.miniscreen import Miniscreen
from PIL import Image, ImageDraw, ImageFont
gpsSerial = serial.Serial('/dev/ttyS0', 9600, timeout=1.)
gpsIO = io.TextIOWrapper(io.BufferedRWPair(gpsSerial, gpsSerial))
gpsSerial.write(b'$PMTK314,0,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1*34\r\n')
ms = Miniscreen()
image = Image.new(
ms.mode,
ms.size,
)
canvas = ImageDraw.Draw(image)
ms.set_max_fps(60)
nmea_list = ['GGA', 'GSA', 'RMC', 'ZDA']
nmea_dict = {
'latitude': "",
'longitude': "",
'speed': "",
'date_time': "",
'satelites': 0,
'altitude': "",
'fix': True,
'fix_type': ""
}
q = queue.Queue()
def readlines():
for line in io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8'):
yield line.strip('\n')
class Robot:
global x_p
global c
rospy.init_node('omoros', anonymous=True)
# fetch /global parameters
param_port = rospy.get_param('~port')
param_baud = rospy.get_param('~baud')
param_modelName = rospy.get_param('~modelName')
# Open Serial port with parameter settings
ser = serial.Serial(param_port, param_baud)
#ser = serial.Serial('/dev/ttyS0', 115200) #For raspberryPi
ser_io = io.TextIOWrapper(io.BufferedRWPair(ser, ser, 1),
newline='\r',
line_buffering=True)
config = VehicleConfig()
pose = MyPose()
goal_1 = MyGoal_1()
goal_2 = MyGoal_2()
joyAxes = []
joyButtons = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] # Buttons 15
joyDeadband = 0.15
exp = 0.3 # Joystick expo setting
isAutoMode = True
isArrowMode = False # Whether to control robo with arrow key or not
arrowCon = ArrowCon
#initialize data
cmd = Command
enc_L = 0.0 # Left wheel encoder count from QENCOD message
enc_R = 0.0 # Right wheel encoder count from QENCOD message
enc_L_prev = 0.0
enc_R_prev = 0.0
enc_offset_L = 0.0
enc_offset_R = 0.0
enc_cnt = 0
odo_L = 0.0 # Left Wheel odometry returned from QODO message
odo_R = 0.0 # Right Wheel odometry returned from QODO message
RPM_L = 0.0 # Left Wheel RPM returned from QRPM message
RPM_R = 0.0 # Right Wheel RPM returned from QRPM message
speedL = 0.0 # Left Wheel speed returned from QDIFF message
speedR = 0.0 # Reft Wheel speed returned from QDIFF message
vel = 0.0 # Velocity returned from CVW command
rot = 0.0 # Rotational speed returned from CVR command
def __init__(self):
## Set vehicle specific configurations
if self.param_modelName == "r1":
print "**********"
print "Driving R1"
print "**********"
self.config.WIDTH = 0.591 # Apply vehicle width for R1 version
self.config.WHEEL_R = 0.11 # Apply wheel radius for R1 version
self.config.WHEEL_MAXV = 1200.0 # Maximum speed can be applied to each wheel (mm/s)
self.config.V_Limit = 0.6 # Limited speed (m/s)
self.config.W_Limit = 0.1
self.config.V_Limit_JOY = 0.25 # Limited speed for joystick control
self.config.W_Limit_JOY = 0.05
self.config.ArrowFwdStep = 250 # Steps move forward based on Odometry
self.config.ArrowRotRate = 0.125
self.config.encoder.Dir = 1.0
self.config.encoder.PPR = 1000
self.config.encoder.GearRatio = 15
elif self.param_modelName == "mini":
print "***************"
print "Driving R1-mini"
print "***************"
self.config.WIDTH = 0.170 # Apply vehicle width for mini version
self.config.WHEEL_R = 0.0336 # Apply wheel radius for mini version
self.config.WHEEL_MAXV = 500.0
self.config.V_Limit = 0.5
self.config.W_Limit = 0.2
self.config.V_Limit_JOY = 0.5
self.config.W_Limit_JOY = 0.1
self.config.ArrowFwdStep = 100
self.config.ArrowRotRate = 0.1
self.config.encoder.Dir = -1.0
self.config.encoder.PPR = 234
self.config.encoder.GearRatio = 21
else:
print "Error: param:modelName, Only support r1 and mini. exit..."
exit()
print('Wheel Track:{:.2f}m, Radius:{:.2f}m'.format(
self.config.WIDTH, self.config.WHEEL_R))
self.config.BodyCircumference = self.config.WIDTH * math.pi
print('Platform Rotation arc length: {:04f}m'.format(
self.config.BodyCircumference))
self.config.WheelCircumference = self.config.WHEEL_R * 2 * math.pi
print('Wheel circumference: {:04f}m'.format(
self.config.WheelCircumference))
self.config.encoder.Step = self.config.WheelCircumference / (
self.config.encoder.PPR * self.config.encoder.GearRatio * 4)
print('Encoder step: {:04f}m/pulse'.format(self.config.encoder.Step))
print(self.ser.name) # Print which port was really used
self.joyAxes = [0, 0, 0, 0, 0, 0, 0, 0]
self.joyButtons = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
# Configure data output
if self.ser.isOpen():
print("Serial Open")
self.resetODO()
sleep(0.05)
self.reset_odometry()
self.setREGI(0, 'QENCOD')
sleep(0.05)
self.setREGI(1, 'QODO')
sleep(0.05)
self.setREGI(2, 'QDIFFV')
sleep(0.05)
self.setREGI(3, '0')
sleep(0.05)
self.setREGI(4, '0')
#self.setREGI(3,'QVW')
#sleep(0.05)
#self.setREGI(4,'QRPM')
sleep(0.05)
self.setSPERI(20)
sleep(0.05)
self.setPEEN(1)
sleep(0.05)
self.reset_odometry()
# Subscriber
rospy.Subscriber("joy", Joy, self.callbackJoy)
rospy.Subscriber('/scan', LaserScan, self.callback)
rospy.Subscriber("cmd_vel", Twist, self.callbackCmdVel)
# publisher
self.pub_enc_l = rospy.Publisher('motor/encoder/left',
Float64,
queue_size=10)
self.pub_enc_r = rospy.Publisher('motor/encoder/right',
Float64,
queue_size=10)
self.pub_motor_status = rospy.Publisher('motor/status',
R1MotorStatusLR,
queue_size=10)
self.odom_pub = rospy.Publisher("odom", Odometry, queue_size=10)
self.odom_broadcaster = TransformBroadcaster()
rate = rospy.Rate(rospy.get_param('~hz', 30)) # 30hz
rospy.Timer(rospy.Duration(0.05), self.joytimer)
rospy.Timer(rospy.Duration(0.01), self.serReader)
#rospy.Timer(rospy.Duration(0.05), self.callback)
self.pose.timestamp = rospy.Time.now()
while not rospy.is_shutdown():
if self.cmd.isAlive == True:
self.cmd.cnt += 1
if self.cmd.cnt > 1000: #Wait for about 3 seconds
self.cmd.isAlive = False
self.isAutoMode = False
rate.sleep()
self.ser.close()
def callback(self, msg):
global x_p
s0 = 0
s1 = 0
s2 = 0
s2 = numpy.mean(msg.ranges[0:24])
s1 = numpy.mean(msg.ranges[108:132])
s0 = numpy.mean(msg.ranges[216:240])
#print('=========================================')
#print(s0)#left
#print(s1)
#print(s2)#right
#print(msg.ranges)
if abs(x_p) < 1:
if (s0 < 0.5) or (s1 < 0.5) or (s2 < 0.5):
a = 0
b = 0
elif (s1 > 0.5):
a = 50
b = 50
else:
a = 0
b = 0
else:
a = 0
b = 0
cmd = '$CDIFFV,{:.0f},{:.0f}'.format(a, b)
self.ser.write(cmd + "\r" + "\n")
def MoveGoal(self, pose, goal, msg):
global x_p
global c
x_p = pose.x
y_p = pose.y
theta_p = pose.theta
x_g = goal.x
y_g = goal.y
theta_g = goal.theta
print(pose.x - goal.x)
if (x_p != x_g):
if abs(x_p) < abs(x_g / 2):
a = 100 * x_p + 25
else:
a = (x_g - x_p) * 100 + 10
if a > 250: a = 250
if a < -250: a = -250
a = a
if -0.01 < (pose.x - goal.x) < 0.01: c = 0
cmd = '$CDIFFV,{:.0f},{:.0f}'.format(a, a)
if self.ser.isOpen():
print cmd
self.ser.write(cmd + "\r" + "\n")
def serReader(self, event):
global x_p
global c
reader = self.ser_io.readline()
if reader:
packet = reader.split(",")
try:
header = packet[0].split("#")[1]
if header.startswith('QVW'):
self.vel = int(packet[1])
self.rot = int(packet[2])
elif header.startswith('QENCOD'):
enc_L = int(packet[1])
enc_R = int(packet[2])
if self.enc_cnt == 0:
self.enc_offset_L = enc_L
self.enc_offset_R = enc_R
self.enc_cnt += 1
self.enc_L = enc_L * self.config.encoder.Dir - self.enc_offset_L
self.enc_R = enc_R * self.config.encoder.Dir - self.enc_offset_R
self.pub_enc_l.publish(Float64(data=self.enc_L))
self.pub_enc_r.publish(Float64(data=self.enc_R))
self.pose = self.updatePose(self.pose, self.enc_L,
self.enc_R)
#print('Encoder:L{:.2f}, R:{:.2f}'.format(self.enc_L, self.enc_R))
elif header.startswith('QODO'):
self.odo_L = float(packet[1]) * self.config.encoder.Dir
self.odo_R = float(packet[2]) * self.config.encoder.Dir
#print('Odo:{:.2f}mm,{:.2f}mm'.format(self.odo_L, self.odo_R))
elif header.startswith('QRPM'):
self.RPM_L = int(packet[1])
self.RPM_R = int(packet[2])
#print('RPM:{:.2f}mm,{:.2f}mm'.format(self.RPM_L, self.RPM_R))
elif header.startswith('QDIFFV'):
self.speedL = int(packet[1])
self.speedR = int(packet[2])
except:
pass
status_left = R1MotorStatus(low_voltage=0,
overloaded=0,
power=0,
encoder=self.enc_L,
RPM=self.RPM_L,
ODO=self.odo_L,
speed=self.speedL)
status_right = R1MotorStatus(low_voltage=0,
overloaded=0,
power=0,
encoder=self.enc_R,
RPM=self.RPM_R,
ODO=self.odo_R,
speed=self.speedR)
self.pub_motor_status.publish(
R1MotorStatusLR(header=Header(stamp=rospy.Time.now()),
Vspeed=self.vel,
Vomega=self.rot,
left=status_left,
right=status_right))
def callbackJoy(self, data):
self.joyAxes = deepcopy(data.axes)
#print('Joy:{:.2f},{:.2f}'.format(self.joyAxes[0], self.joyAxes[1]))
# Read the most recent button state
newJoyButtons = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
newJoyButtons = deepcopy(data.buttons)
# Check if button 1(B) is newly set
if (newJoyButtons[1]
== 1) and (newJoyButtons[1] != self.joyButtons[1]):
if self.isAutoMode != True:
self.isAutoMode = True
print "In Auto mode"
else:
self.isAutoMode = False
print "In Manual mode"
if (newJoyButtons[10]
== 1) and (newJoyButtons[10] != self.joyButtons[10]):
if self.isArrowMode != True:
self.isArrowMode = True
self.arrowCon.isFinished = True
print "Joystick Arrow Mode"
else:
self.isArrowMode = False
print "Joystick Axis Mode"
if self.isArrowMode == True:
#if (newJoyButtons[13]==1) or (newJoyButtons[14]==1):
#if (self.joyAxes[7]==1.0) or (self.joyAxes[7]==-1.0):
if (self.joyAxes[5] == 1.0) or (self.joyAxes[5] == -1.0):
if self.arrowCon.isFinished == True:
self.arrowCon.isFinished = False
#if newJoyButtons[13]==1: # FWD arrow
#if self.joyAxes[7]==1.0:
if self.joyAxes[5] == 1.0:
self.arrowCommand("FWD", self.arrowCon, self.config)
else:
self.arrowCommand("REAR", self.arrowCon, self.config)
#print "Arrow: {:.2f} {:.2f} ".format(self.arrowCon.startOdo_L, self.arrowCon.targetOdo_L)
#elif (newJoyButtons[11]==1) or (newJoyButtons[12]==1): #For Xbox360 controller
#elif (self.joyAxes[6]==1.0) or (self.joyAxes[6]==-1.0):
elif (self.joyAxes[4] == 1.0) or (self.joyAxes[4] == -1.0):
if self.arrowCon.isFinished == True:
turnRate = 10.5
self.arrowCon.isFinished = False
#if newJoyButtons[11]==1: # Left arrow
#if self.joyAxes[6]==1.0:
if self.joyAxes[4] == 1.0:
self.arrowCommand("LEFT", self.arrowCon, self.config)
else: # Right arrow
self.arrowCommand("RIGHT", self.arrowCon, self.config)
# Update button state
self.joyButtons = deepcopy(newJoyButtons)
def arrowCommand(self, command, arrowCon, config):
if command == "FWD":
arrowCon.setFwd = 1
arrowCon.targetOdo_L = self.odo_L + config.ArrowFwdStep #target 1 step ahead
arrowCon.targetOdo_R = self.odo_R + config.ArrowFwdStep #target 1 step ahead
print "Arrow Fwd"
elif command == "REAR":
self.arrowCon.setFwd = -1
self.arrowCon.targetOdo_L = self.odo_L - self.config.ArrowFwdStep #target 1 step rear
self.arrowCon.targetOdo_R = self.odo_R - self.config.ArrowFwdStep #target 1 step rear
print "Arrow Rev"
elif command == "LEFT":
arrowCon.setRot = 1
arrowCon.targetOdo_L = self.odo_L - config.BodyCircumference * 1000 * config.ArrowRotRate
arrowCon.targetOdo_R = self.odo_R + config.BodyCircumference * 1000 * config.ArrowRotRate
print "Arrow Left"
elif command == "RIGHT":
arrowCon.setRot = -1
arrowCon.targetOdo_L = self.odo_L + config.BodyCircumference * 1000 * config.ArrowRotRate
arrowCon.targetOdo_R = self.odo_R - config.BodyCircumference * 1000 * config.ArrowRotRate
print "Arrow Right"
def callbackCmdVel(self, cmd):
""" Set wheel speed from cmd message from auto navigation """
if self.isAutoMode:
#print "CMD_VEL: {:.2f} {:.2f} ".format(cmd.linear.x, cmd.angular.z)
cmdV = cmd.linear.x
cmdW = cmd.angular.z
if cmdV > self.config.V_Limit:
cmdV = self.config.V_Limit
elif cmdV < -self.config.V_Limit:
cmdV = -self.config.V_Limit
if cmdW > self.config.W_Limit:
cmdW = self.config.W_Limit
elif cmdW < -self.config.W_Limit:
cmdW = -self.config.W_Limit
(speedL, speedR) = self.getWheelSpeedLR(self.config, cmdV, cmdW)
#print "SPEED LR: {:.2f} {:.2f} ".format(speedL, speedR)
self.sendCDIFFVcontrol(speedL * 200, speedR * 200)
def reset_odometry(self):
self.last_speedL = 0.0
self.last_speedR = 0.0
def joytimer(self, event):
if not self.isAutoMode:
self.joy_v = self.joyAxes[1]
self.joy_w = self.joyAxes[0]
#print "Joy mode: {:.2f} {:.2f} ".format(self.joy_v, self.joy_w)
else:
return
if not self.isArrowMode:
# Apply joystick deadband and calculate vehicle speed (mm/s) and rate of chage of orientation(rad/s)
joyV = 0.0
joyR = 0.0
if abs(self.joy_v) < self.joyDeadband:
joyV = 0.0
else:
joyV = (1 - self.exp) * self.joy_v + (
self.exp) * self.joy_v * self.joy_v * self.joy_v
if abs(self.joy_w) < self.joyDeadband:
joyR = 0.0
else:
joyR = (1 - self.exp) * self.joy_w + (
self.exp) * self.joy_w * self.joy_w * self.joy_w
# Apply max Vehicle speed
(speedL,
speedR) = self.getWheelSpeedLR(self.config,
joyV * self.config.V_Limit_JOY,
joyR * self.config.W_Limit_JOY)
#print "Joystick VL, VR: {:.2f} {:.2f}".format(speedL, speedR)
self.sendCDIFFVcontrol(speedL * 1000, speedR * 1000)
else:
if self.arrowCon.isFinished == False:
if self.arrowCon.setFwd == 1: # For forward motion
if (self.odo_L < self.arrowCon.targetOdo_L) or (
self.odo_R < self.arrowCon.targetOdo_R):
#print "Fwd: {:.2f} {:.2f} ".format(self.odo_L, self.odo_R)
self.sendCDIFFVcontrol(100, 100)
else:
self.sendCDIFFVcontrol(0, 0)
self.arrowCon.isFinished = True
self.arrowCon.setFwd = 0
print "Finished!"
elif self.arrowCon.setFwd == -1:
if (self.odo_L > self.arrowCon.targetOdo_L) or (
self.odo_R > self.arrowCon.targetOdo_R):
#print "Rev: {:.2f} {:.2f} ".format(self.odo_L, self.odo_R)
self.sendCDIFFVcontrol(-100, -100)
else:
self.sendCDIFFVcontrol(0, 0)
self.arrowCon.isFinished = True
self.arrowCon.setFwd = 0
print "Finished!"
elif self.arrowCon.setRot == 1: #Turning left
if (self.odo_L > self.arrowCon.targetOdo_L) or (
self.odo_R < self.arrowCon.targetOdo_R):
#print "Left: {:.2f} {:.2f} ".format(self.odo_L, self.odo_R)
self.sendCDIFFVcontrol(-100, 100)
else:
self.sendCDIFFVcontrol(0, 0)
self.arrowCon.isFinished = True
self.arrowCon.setRot = 0
print "Finished!"
elif self.arrowCon.setRot == -1: #Turning Right
if (self.odo_L < self.arrowCon.targetOdo_L) or (
self.odo_R > self.arrowCon.targetOdo_R):
#print "Right: {:.2f} {:.2f} ".format(self.odo_L, self.odo_R)
self.sendCDIFFVcontrol(100, -100)
else:
self.sendCDIFFVcontrol(0, 0)
self.arrowCon.isFinished = True
self.arrowCon.setRot = 0
print "Finished!"
def updatePose(self, pose, encoderL, encoderR):
"""Update Position x,y,theta from encoder count L, R
Return new Position x,y,theta"""
global x_p
print c
now = rospy.Time.now()
dL = encoderL - self.enc_L_prev
dR = encoderR - self.enc_R_prev
self.enc_L_prev = encoderL
self.enc_R_prev = encoderR
dT = (now - pose.timestamp) / 1000000.0
pose.timestamp = now
x = pose.x
y = pose.y
theta = pose.theta
R = 0.0
if (dR - dL) == 0:
R = 0.0
else:
R = self.config.WIDTH / 2.0 * (dL + dR) / (dR - dL)
Wdt = (dR - dL) * self.config.encoder.Step / self.config.WIDTH
ICCx = x - R * np.sin(theta)
ICCy = y + R * np.cos(theta)
pose.x = np.cos(Wdt) * (x - ICCx) - np.sin(Wdt) * (y - ICCy) + ICCx
pose.y = np.sin(Wdt) * (x - ICCx) + np.cos(Wdt) * (y - ICCy) + ICCy
pose.theta = theta + Wdt
twist = TwistWithCovariance()
twist.twist.linear.x = self.vel / 1000.0
twist.twist.linear.y = 0.0
twist.twist.angular.z = self.rot / 1000.0
Vx = twist.twist.linear.x
Vy = twist.twist.linear.y
Vth = twist.twist.angular.z
odom_quat = quaternion_from_euler(0, 0, pose.theta)
self.odom_broadcaster.sendTransform((pose.x, pose.y, 0.), odom_quat,
now, 'base_link', 'odom')
#self.odom_broadcaster.sendTransform((pose.x,pose.y,0.),odom_quat,now,'base_footprint','odom')
odom = Odometry()
odom.header.stamp = now
odom.header.frame_id = 'odom'
odom.pose.pose = Pose(Point(pose.x, pose.y, 0.),
Quaternion(*odom_quat))
odom.child_frame_id = 'base_link'
#odom.child_frame_id = 'base_footprint'
odom.twist.twist = Twist(Vector3(Vx, Vy, 0), Vector3(0, 0, Vth))
print "x:{:.2f} y:{:.2f} theta:{:.2f}".format(
pose.x, pose.y, pose.theta * 180 / math.pi)
self.odom_pub.publish(odom)
x_p = pose.x
return pose
def getWheelSpeedLR(self, config, V_m_s, W_rad_s):
"""Takes Speed V (m/s) and Rotational sepeed W(rad/s) and compute each wheel speed in m/s
Kinematics reference from http://enesbot.me/kinematic-model-of-a-differential-drive-robot.html"""
speedL = V_m_s - config.WIDTH * W_rad_s / config.WHEEL_R / 2.0
speedR = V_m_s + config.WIDTH * W_rad_s / config.WHEEL_R / 2.0
return speedL, speedR
def sendCVWcontrol(self, config, V_mm_s, W_mrad_s):
""" Set Vehicle velocity and rotational speed """
if V_mm_s > config.V_Limit:
V_mm_s = config.V_Limit
elif V_mm_s < -config.V_Limit:
V_mm_s = -config.V_limit
if W_mrad_s > config.W_Limit:
W_mrad_s = config.W_Limit
elif W_mrad_s < -config.W_Limit:
W_mrad_s = -config.W_Limit
# Make a serial message to be sent to motor driver unit
cmd = '$CVW,{:.0f},{:.0f}'.format(V_mm_s, W_mrad_s)
print cmd
if self.ser.isOpen():
print cmd
self.ser.write(cmd + "\r" + "\n")
def sendCDIFFVcontrol(self, VLmm_s, VRmm_s):
""" Set differential wheel speed for Left and Right """
if VLmm_s > self.config.WHEEL_MAXV:
VLmm_s = self.config.WHEEL_MAXV
elif VLmm_s < -self.config.WHEEL_MAXV:
VLmm_s = -self.config.WHEEL_MAXV
if VRmm_s > self.config.WHEEL_MAXV:
VRmm_s = self.config.WHEEL_MAXV
elif VRmm_s < -self.config.WHEEL_MAXV:
VRmm_s = -self.config.WHEEL_MAXV
# Make a serial message to be sent to motor driver unit
cmd = '$CDIFFV,{:.0f},{:.0f}'.format(VLmm_s, VRmm_s)
if self.ser.isOpen():
#print cmd
self.ser.write(cmd + "\r" + "\n")
def setREGI(self, param1, param2):
msg = "$SREGI," + str(param1) + ',' + param2
self.ser.write(msg + "\r" + "\n")
def setSPERI(self, param):
msg = "$SPERI," + str(param)
self.ser.write(msg + "\r" + "\n")
def setPEEN(self, param):
msg = "$SPEEN," + str(param)
self.ser.write(msg + "\r" + "\n")
def resetODO(self):
self.ser.write("$SODO\r\n")
def main(
argsl=None, # type: List[str]
args=None, # type: argparse.Namespace
job_order_object=None, # type: MutableMapping[Text, Any]
stdin=sys.stdin, # type: IO[Any]
stdout=None, # type: Union[TextIO, StreamWriter]
stderr=sys.stderr, # type: IO[Any]
versionfunc=versionstring, # type: Callable[[], Text]
logger_handler=None, #
custom_schema_callback=None, # type: Callable[[], None]
executor=None, # type: Callable[..., Tuple[Dict[Text, Any], Text]]
loadingContext=None, # type: LoadingContext
runtimeContext=None # type: RuntimeContext
): # type: (...) -> int
if not stdout: # force UTF-8 even if the console is configured differently
if (hasattr(sys.stdout, "encoding") # type: ignore
and sys.stdout.encoding != 'UTF-8'): # type: ignore
if PY3 and hasattr(sys.stdout, "detach"):
stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
else:
stdout = getwriter('utf-8')(sys.stdout) # type: ignore
else:
stdout = cast(TextIO, sys.stdout) # type: ignore
_logger.removeHandler(defaultStreamHandler)
if logger_handler is not None:
stderr_handler = logger_handler
else:
stderr_handler = logging.StreamHandler(stderr)
_logger.addHandler(stderr_handler)
# pre-declared for finally block
workflowobj = None
prov_log_handler = None # type: Optional[logging.StreamHandler]
try:
if args is None:
if argsl is None:
argsl = sys.argv[1:]
args = arg_parser().parse_args(argsl)
if args.record_container_id:
if not args.cidfile_dir:
args.cidfile_dir = os.getcwd()
del args.record_container_id
if runtimeContext is None:
runtimeContext = RuntimeContext(vars(args))
else:
runtimeContext = runtimeContext.copy()
# If on Windows platform, a default Docker Container is used if not
# explicitely provided by user
if onWindows() and not runtimeContext.default_container:
# This docker image is a minimal alpine image with bash installed
# (size 6 mb). source: https://github.com/frol/docker-alpine-bash
runtimeContext.default_container = windows_default_container_id
# If caller parsed its own arguments, it may not include every
# cwltool option, so fill in defaults to avoid crashing when
# dereferencing them in args.
for key, val in iteritems(get_default_args()):
if not hasattr(args, key):
setattr(args, key, val)
# Configure logging
rdflib_logger = logging.getLogger("rdflib.term")
rdflib_logger.addHandler(stderr_handler)
rdflib_logger.setLevel(logging.ERROR)
if args.quiet:
# Silence STDERR, not an eventual provenance log file
stderr_handler.setLevel(logging.WARN)
if runtimeContext.debug:
# Increase to debug for both stderr and provenance log file
_logger.setLevel(logging.DEBUG)
rdflib_logger.setLevel(logging.DEBUG)
formatter = None # type: Optional[logging.Formatter]
if args.timestamps:
formatter = logging.Formatter("[%(asctime)s] %(message)s",
"%Y-%m-%d %H:%M:%S")
stderr_handler.setFormatter(formatter)
##
if args.version:
print(versionfunc())
return 0
_logger.info(versionfunc())
if args.print_supported_versions:
print("\n".join(supported_cwl_versions(args.enable_dev)))
return 0
if not args.workflow:
if os.path.isfile("CWLFile"):
setattr(args, "workflow", "CWLFile")
else:
_logger.error("")
_logger.error(
"CWL document required, no input file was provided")
arg_parser().print_help()
return 1
if args.relax_path_checks:
command_line_tool.ACCEPTLIST_RE = command_line_tool.ACCEPTLIST_EN_RELAXED_RE
if args.ga4gh_tool_registries:
ga4gh_tool_registries[:] = args.ga4gh_tool_registries
if not args.enable_ga4gh_tool_registry:
del ga4gh_tool_registries[:]
if custom_schema_callback is not None:
custom_schema_callback()
elif args.enable_ext:
res = pkg_resources.resource_stream(__name__, 'extensions.yml')
use_custom_schema("v1.0", "http://commonwl.org/cwltool",
res.read())
res.close()
else:
use_standard_schema("v1.0")
if args.provenance:
if not args.compute_checksum:
_logger.error(
"--provenance incompatible with --no-compute-checksum")
return 1
ro = ResearchObject(temp_prefix_ro=args.tmpdir_prefix,
orcid=args.orcid,
full_name=args.cwl_full_name)
runtimeContext.research_obj = ro
log_file_io = ro.open_log_file_for_activity(ro.engine_uuid)
prov_log_handler = logging.StreamHandler(log_file_io)
class ProvLogFormatter(logging.Formatter):
"""Enforce ISO8601 with both T and Z."""
def __init__(self): # type: () -> None
super(ProvLogFormatter,
self).__init__("[%(asctime)sZ] %(message)s")
def formatTime(self, record, datefmt=None):
# type: (logging.LogRecord, str) -> str
record_time = time.gmtime(record.created)
formatted_time = time.strftime("%Y-%m-%dT%H:%M:%S",
record_time)
with_msecs = "%s,%03d" % (formatted_time, record.msecs)
return with_msecs
prov_log_handler.setFormatter(ProvLogFormatter())
_logger.addHandler(prov_log_handler)
_logger.debug(u"[provenance] Logging to %s", log_file_io)
if argsl is not None:
# Log cwltool command line options to provenance file
_logger.info("[cwltool] %s %s", sys.argv[0], u" ".join(argsl))
_logger.debug(u"[cwltool] Arguments: %s", args)
if loadingContext is None:
loadingContext = LoadingContext(vars(args))
else:
loadingContext = loadingContext.copy()
loadingContext.loader = default_loader(
loadingContext.fetcher_constructor)
loadingContext.research_obj = runtimeContext.research_obj
loadingContext.disable_js_validation = \
args.disable_js_validation or (not args.do_validate)
loadingContext.construct_tool_object = getdefault(
loadingContext.construct_tool_object, workflow.default_make_tool)
loadingContext.resolver = getdefault(loadingContext.resolver,
tool_resolver)
loadingContext.do_update = not (args.pack or args.print_subgraph)
uri, tool_file_uri = resolve_tool_uri(
args.workflow,
resolver=loadingContext.resolver,
fetcher_constructor=loadingContext.fetcher_constructor)
try_again_msg = "" if args.debug else ", try again with --debug for more information"
try:
job_order_object, input_basedir, jobloader = load_job_order(
args, stdin, loadingContext.fetcher_constructor,
loadingContext.overrides_list, tool_file_uri)
if args.overrides:
loadingContext.overrides_list.extend(
load_overrides(file_uri(os.path.abspath(args.overrides)),
tool_file_uri))
loadingContext, workflowobj, uri = fetch_document(
uri, loadingContext)
assert loadingContext.loader is not None
if args.print_deps:
printdeps(workflowobj, loadingContext.loader, stdout,
args.relative_deps, uri)
return 0
loadingContext, uri \
= resolve_and_validate_document(loadingContext, workflowobj, uri,
preprocess_only=(args.print_pre or args.pack),
skip_schemas=args.skip_schemas)
assert loadingContext.loader is not None
processobj, metadata = loadingContext.loader.resolve_ref(uri)
processobj = cast(CommentedMap, processobj)
if args.pack:
stdout.write(
print_pack(loadingContext.loader, processobj, uri,
metadata))
return 0
if args.provenance and runtimeContext.research_obj:
# Can't really be combined with args.pack at same time
runtimeContext.research_obj.packed_workflow(
print_pack(loadingContext.loader, processobj, uri,
metadata))
if args.print_pre:
stdout.write(
json_dumps(processobj,
indent=4,
sort_keys=True,
separators=(',', ': ')))
return 0
tool = make_tool(uri, loadingContext)
if args.make_template:
def my_represent_none(self, data): # pylint: disable=unused-argument
"""Force clean representation of 'null'."""
return self.represent_scalar(u'tag:yaml.org,2002:null',
u'null')
yaml.RoundTripRepresenter.add_representer(
type(None), my_represent_none)
yaml.round_trip_dump(generate_input_template(tool),
sys.stdout,
default_flow_style=False,
indent=4,
block_seq_indent=2)
return 0
if args.validate:
print("{} is valid CWL.".format(args.workflow))
return 0
if args.print_rdf:
stdout.write(
printrdf(tool, loadingContext.loader.ctx,
args.rdf_serializer))
return 0
if args.print_dot:
printdot(tool, loadingContext.loader.ctx, stdout)
return 0
if args.print_targets:
for f in ("outputs", "steps", "inputs"):
if tool.tool[f]:
_logger.info("%s%s targets:", f[0].upper(), f[1:-1])
stdout.write(" " + "\n ".join(
[shortname(t["id"]) for t in tool.tool[f]]) + "\n")
return 0
if args.target:
if isinstance(tool, Workflow):
url = urllib.parse.urlparse(tool.tool["id"])
if url.fragment:
extracted = get_subgraph(
[tool.tool["id"] + "/" + r for r in args.target],
tool)
else:
extracted = get_subgraph([
loadingContext.loader.fetcher.urljoin(
tool.tool["id"], "#" + r) for r in args.target
], tool)
else:
_logger.error("Can only use --target on Workflows")
return 1
loadingContext.loader.idx[extracted["id"]] = extracted
tool = make_tool(extracted["id"], loadingContext)
if args.print_subgraph:
if "name" in tool.tool:
del tool.tool["name"]
stdout.write(
json_dumps(tool.tool,
indent=4,
sort_keys=True,
separators=(',', ': ')))
return 0
except (validate.ValidationException) as exc:
_logger.error(u"Tool definition failed validation:\n%s",
exc,
exc_info=args.debug)
return 1
except (RuntimeError, WorkflowException) as exc:
_logger.error(u"Tool definition failed initialization:\n%s",
exc,
exc_info=args.debug)
return 1
except Exception as exc:
_logger.error(
u"I'm sorry, I couldn't load this CWL file%s.\nThe error was: %s",
try_again_msg,
exc if not args.debug else "",
exc_info=args.debug)
return 1
if isinstance(tool, int):
return tool
# If on MacOS platform, TMPDIR must be set to be under one of the
# shared volumes in Docker for Mac
# More info: https://dockstore.org/docs/faq
if sys.platform == "darwin":
default_mac_path = "/private/tmp/docker_tmp"
if runtimeContext.tmp_outdir_prefix == DEFAULT_TMP_PREFIX:
runtimeContext.tmp_outdir_prefix = default_mac_path
if runtimeContext.tmpdir_prefix == DEFAULT_TMP_PREFIX:
runtimeContext.tmpdir_prefix = default_mac_path
for dirprefix in ("tmpdir_prefix", "tmp_outdir_prefix", "cachedir"):
if getattr(runtimeContext, dirprefix) and getattr(
runtimeContext, dirprefix) != DEFAULT_TMP_PREFIX:
sl = "/" if getattr(runtimeContext, dirprefix).endswith("/") or dirprefix == "cachedir" \
else ""
setattr(
runtimeContext, dirprefix,
os.path.abspath(getattr(runtimeContext, dirprefix)) + sl)
if not os.path.exists(
os.path.dirname(getattr(runtimeContext, dirprefix))):
try:
os.makedirs(
os.path.dirname(getattr(runtimeContext,
dirprefix)))
except Exception as e:
_logger.error("Failed to create directory: %s", e)
return 1
if args.cachedir:
if args.move_outputs == "move":
runtimeContext.move_outputs = "copy"
runtimeContext.tmp_outdir_prefix = args.cachedir
runtimeContext.secret_store = getdefault(runtimeContext.secret_store,
SecretStore())
runtimeContext.make_fs_access = getdefault(
runtimeContext.make_fs_access, StdFsAccess)
try:
initialized_job_order_object = init_job_order(
job_order_object,
args,
tool,
jobloader,
stdout,
print_input_deps=args.print_input_deps,
relative_deps=args.relative_deps,
make_fs_access=runtimeContext.make_fs_access,
input_basedir=input_basedir,
secret_store=runtimeContext.secret_store)
except SystemExit as err:
return err.code
if not executor:
if args.parallel:
executor = MultithreadedJobExecutor()
runtimeContext.select_resources = executor.select_resources
else:
executor = SingleJobExecutor()
assert executor is not None
try:
runtimeContext.basedir = input_basedir
del args.workflow
del args.job_order
conf_file = getattr(args,
"beta_dependency_resolvers_configuration",
None) # Text
use_conda_dependencies = getattr(args, "beta_conda_dependencies",
None) # Text
if conf_file or use_conda_dependencies:
runtimeContext.job_script_provider = DependenciesConfiguration(
args)
else:
runtimeContext.find_default_container = functools.partial(
find_default_container,
default_container=runtimeContext.default_container,
use_biocontainers=args.beta_use_biocontainers)
(out, status) = executor(tool,
initialized_job_order_object,
runtimeContext,
logger=_logger)
if out is not None:
if runtimeContext.research_obj is not None:
runtimeContext.research_obj.create_job(out, None, True)
def loc_to_path(obj):
for field in ("path", "nameext", "nameroot", "dirname"):
if field in obj:
del obj[field]
if obj["location"].startswith("file://"):
obj["path"] = uri_file_path(obj["location"])
visit_class(out, ("File", "Directory"), loc_to_path)
# Unsetting the Generation from final output object
visit_class(out, ("File", ),
MutationManager().unset_generation)
if isinstance(out, string_types):
stdout.write(out)
else:
stdout.write(
json_dumps(
out,
indent=4, # type: ignore
ensure_ascii=False))
stdout.write("\n")
if hasattr(stdout, "flush"):
stdout.flush() # type: ignore
if status != "success":
_logger.warning(u"Final process status is %s", status)
return 1
_logger.info(u"Final process status is %s", status)
return 0
except (validate.ValidationException) as exc:
_logger.error(u"Input object failed validation:\n%s",
exc,
exc_info=args.debug)
return 1
except UnsupportedRequirement as exc:
_logger.error(u"Workflow or tool uses unsupported feature:\n%s",
exc,
exc_info=args.debug)
return 33
except WorkflowException as exc:
_logger.error(u"Workflow error%s:\n%s",
try_again_msg,
strip_dup_lineno(Text(exc)),
exc_info=args.debug)
return 1
except Exception as exc: # pylint: disable=broad-except
_logger.error(u"Unhandled error%s:\n %s",
try_again_msg,
exc,
exc_info=args.debug)
return 1
finally:
if args and runtimeContext and runtimeContext.research_obj \
and workflowobj:
research_obj = runtimeContext.research_obj
assert loadingContext is not None
assert loadingContext.loader is not None
prov_dependencies = prov_deps(workflowobj, loadingContext.loader,
uri)
research_obj.generate_snapshot(prov_dependencies)
if prov_log_handler is not None:
# Stop logging so we won't half-log adding ourself to RO
_logger.debug(u"[provenance] Closing provenance log file %s",
prov_log_handler)
_logger.removeHandler(prov_log_handler)
# Ensure last log lines are written out
prov_log_handler.flush()
# Underlying WritableBagFile will add the tagfile to the manifest
prov_log_handler.stream.close()
prov_log_handler.close()
research_obj.close(args.provenance)
_logger.removeHandler(stderr_handler)
_logger.addHandler(defaultStreamHandler)
def p(*c):
print(*c)
# _*_ coding:utf-8 _*_
import pandas as pd
import numpy as np
import io
import sys
sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf8')
def loadDataSet():
return [[1, 3, 4], [2, 3, 5], [1, 2, 3, 5], [2, 5]]
def createC1(dataSet):
C1 = []
for transaction in dataSet:
for item in transaction:
if not [item] in C1:
C1.append([item])
# C1.sort()
return list(map(frozenset, C1))#use frozen set so we
#can use it as a key in a dict
def scanD(D, Ck, minSupport):
ssCnt = {}
for tid in D:
for can in Ck:
if can.issubset(tid):
if can not in ssCnt:
ssCnt[can]=1
else:
# -*- coding: utf-8 -*-
import re
import json
import base64
import binascii
import rsa
import requests
import io
# for test
import sys
sys.path.insert(0, '..')
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='gb18030')
from config import WBCLIENT, user_agent
from config import USER_NAME, PASSWD
from logger import logger
session = requests.session()
session.headers['User-Agent'] = user_agent
def encrypt_passwd(passwd, pubkey, servertime, nonce): #rsa加密密码
key = rsa.PublicKey(int(pubkey, 16), int(
'10001',
16)) #int('10001', 16) 并非要转换成16进制，而是说10001是16 进制的数，int函数要将其转化为十进制
message = str(servertime) + '\t' + str(nonce) + '\n' + str(passwd)
passwd = rsa.encrypt(message.encode('utf-8'), key)
return binascii.b2a_hex(passwd)
# 登录微博的流程
def split_wet_file(wet_file_path, counter_inc_fn=None):
"""Split a WET file into separate pages."""
logging.info("Splitting file: %s", wet_file_path)
if not counter_inc_fn:
counter_inc_fn = get_counter_inc_fn("split-wet-file")
counter_inc_fn("wet-file")
with tf.io.gfile.GFile(wet_file_path, "rb") as f, gzip.GzipFile(
fileobj=f) as g:
url = None
content = None
content_len = None
content_type = None
timestamp = None
def _maybe_get_page():
"""Generate a (url, {features}) page."""
if not url and url is not None:
counter_inc_fn("page-filitered-nourl")
if not content and content is not None:
counter_inc_fn("page-filtered-nocontent")
if not content_type and content_type is not None:
counter_inc_fn("page-nocontenttype")
if not content_len and content_len is not None:
counter_inc_fn("page-nocontentlen")
if not timestamp and timestamp is not None:
counter_inc_fn("page-notimestamp")
if content and url:
counter_inc_fn("page-emitted")
return (url, {
"text": "\n".join(content),
"content-type": content_type,
"content-length": content_len,
"timestamp": timestamp,
"url": url
})
return None
for line in io.TextIOWrapper(g, encoding="utf-8"):
line = line.strip()
if not line:
continue
if line == _PAGE_DELIMITER:
page = _maybe_get_page()
if page:
yield page
url = ""
content = []
content_len = ""
content_type = ""
timestamp = ""
if line.startswith(_URL_KEY):
url = line[len(_URL_KEY):].strip()
if line.startswith(_URL_DATE):
timestamp = line[len(_URL_DATE):].strip()
if line.startswith(_CONTENT_TYPE):
content_type = line[len(_CONTENT_TYPE):].strip()
if line.startswith(_CONTENT_LEN):
content_len = line[len(_CONTENT_LEN):].strip()
if line.startswith(_METADATA_PREFIXES):
continue
content.append(line)
page = _maybe_get_page()
if page:
yield page
def _set_output(self, outfiles: OutputFiles):
self._filters = []
self._outfiles = outfiles
filter_wrapper = self._filter_wrapper()
for filter_class, outfile in (
(RestFileWriter, outfiles.rest),
(InfoFileWriter, outfiles.info),
(WildcardFileWriter, outfiles.wildcard),
):
if outfile:
textiowrapper = io.TextIOWrapper(outfile)
self._textiowrappers.append(textiowrapper)
self._filters.append(filter_wrapper(None, filter_class(textiowrapper), None))
# minimum length and maximum length
for lengths, file1, file2, filter_class in (
(self._minimum_length, outfiles.too_short, outfiles.too_short2, TooShortReadFilter),
(self._maximum_length, outfiles.too_long, outfiles.too_long2, TooLongReadFilter)
):
if lengths is None:
continue
writer = self._open_writer(file1, file2) if file1 else None
f1 = filter_class(lengths[0]) if lengths[0] is not None else None
if len(lengths) == 2 and lengths[1] is not None:
f2 = filter_class(lengths[1])
else:
f2 = None
self._filters.append(filter_wrapper(writer, filter=f1, filter2=f2))
if self.max_n is not None:
f1 = f2 = NContentFilter(self.max_n)
self._filters.append(filter_wrapper(None, f1, f2))
if self.discard_casava:
f1 = f2 = CasavaFilter()
self._filters.append(filter_wrapper(None, f1, f2))
if int(self.discard_trimmed) + int(self.discard_untrimmed) + int(outfiles.untrimmed is not None) > 1:
raise ValueError('discard_trimmed, discard_untrimmed and outfiles.untrimmed must not '
'be set simultaneously')
if outfiles.demultiplex:
self._demultiplexer = self._create_demultiplexer(outfiles)
self._filters.append(self._demultiplexer)
else:
# Allow overriding the wrapper for --discard-untrimmed/--untrimmed-(paired-)output
untrimmed_filter_wrapper = self._untrimmed_filter_wrapper()
# Set up the remaining filters to deal with --discard-trimmed,
# --discard-untrimmed and --untrimmed-output. These options
# are mutually exclusive in order to avoid brain damage.
if self.discard_trimmed:
self._filters.append(
filter_wrapper(None, DiscardTrimmedFilter(), DiscardTrimmedFilter()))
elif self.discard_untrimmed:
self._filters.append(
untrimmed_filter_wrapper(None, DiscardUntrimmedFilter(), DiscardUntrimmedFilter()))
elif outfiles.untrimmed:
untrimmed_writer = self._open_writer(outfiles.untrimmed, outfiles.untrimmed2)
self._filters.append(
untrimmed_filter_wrapper(untrimmed_writer, DiscardUntrimmedFilter(), DiscardUntrimmedFilter()))
self._filters.append(self._final_filter(outfiles))
def create():
fa = _make_argument_optional(fs.open_output_stream, metadata=None)(path)
fp = FileProxy(fa, path, lambda: fs.open_output_stream(path))
return io.TextIOWrapper(fa, encoding=encoding)
def reader(buffer, title=None):
print(f'{title}: ', ' '.join((io.TextIOWrapper(buffer.stderr,
encoding='utf-8'))))
return
import gzip, io, csv
import numpy as np
import matplotlib.pyplot as plt
from sklearn.neural_network import MLPClassifier
from sklearn.externals import joblib
from sklearn.decomposition import PCA
from sklearn import metrics
X, y = [], []
with io.TextIOWrapper(gzip.open("../data/train.csv.gz", "r")) as csvfile:
csvreader = csv.reader(csvfile, delimiter=',', quotechar='|')
for i, row in enumerate(csvreader):
if i == 0: continue
vals = row[1:]
X.append(vals)
y.append([row[0]])
X = np.array(X, dtype=float)
X = X / 255
y = np.array(y, dtype=str)
# plot the first 10 images
def gimg(i):
# images must be 28x28x3
return np.reshape(
# greyscale images using the same value for R G B
np.column_stack((X[i], X[i], X[i])),
(28, 28, 3))
def isolation(self, input=None, env=None, color=False):
"""A context manager that sets up the isolation for invoking of a
command line tool. This sets up stdin with the given input data
and `os.environ` with the overrides from the given dictionary.
This also rebinds some internals in Click to be mocked (like the
prompt functionality).
This is automatically done in the :meth:`invoke` method.
.. versionadded:: 4.0
The ``color`` parameter was added.
:param input: the input stream to put into sys.stdin.
:param env: the environment overrides as dictionary.
:param color: whether the output should contain color codes. The
application can still override this explicitly.
"""
input = make_input_stream(input, self.charset)
old_stdin = sys.stdin
old_stdout = sys.stdout
old_stderr = sys.stderr
old_forced_width = formatting.FORCED_WIDTH
formatting.FORCED_WIDTH = 80
env = self.make_env(env)
if PY2:
bytes_output = StringIO()
if self.echo_stdin:
input = EchoingStdin(input, bytes_output)
sys.stdout = bytes_output
if not self.mix_stderr:
bytes_error = StringIO()
sys.stderr = bytes_error
else:
bytes_output = io.BytesIO()
if self.echo_stdin:
input = EchoingStdin(input, bytes_output)
input = io.TextIOWrapper(input, encoding=self.charset)
sys.stdout = io.TextIOWrapper(bytes_output, encoding=self.charset)
if not self.mix_stderr:
bytes_error = io.BytesIO()
sys.stderr = io.TextIOWrapper(bytes_error,
encoding=self.charset)
if self.mix_stderr:
sys.stderr = sys.stdout
sys.stdin = input
def visible_input(prompt=None):
sys.stdout.write(prompt or "")
val = input.readline().rstrip("\r\n")
sys.stdout.write("{}\n".format(val))
sys.stdout.flush()
return val
def hidden_input(prompt=None):
sys.stdout.write("{}\n".format(prompt or ""))
sys.stdout.flush()
return input.readline().rstrip("\r\n")
def _getchar(echo):
char = sys.stdin.read(1)
if echo:
sys.stdout.write(char)
sys.stdout.flush()
return char
default_color = color
def should_strip_ansi(stream=None, color=None):
if color is None:
return not default_color
return not color
old_visible_prompt_func = termui.visible_prompt_func
old_hidden_prompt_func = termui.hidden_prompt_func
old__getchar_func = termui._getchar
old_should_strip_ansi = utils.should_strip_ansi
termui.visible_prompt_func = visible_input
termui.hidden_prompt_func = hidden_input
termui._getchar = _getchar
utils.should_strip_ansi = should_strip_ansi
old_env = {}
try:
for key, value in iteritems(env):
old_env[key] = os.environ.get(key)
if value is None:
try:
del os.environ[key]
except Exception:
pass
else:
os.environ[key] = value
yield (bytes_output, not self.mix_stderr and bytes_error)
finally:
for key, value in iteritems(old_env):
if value is None:
try:
del os.environ[key]
except Exception:
pass
else:
os.environ[key] = value
sys.stdout = old_stdout
sys.stderr = old_stderr
sys.stdin = old_stdin
termui.visible_prompt_func = old_visible_prompt_func
termui.hidden_prompt_func = old_hidden_prompt_func
termui._getchar = old__getchar_func
utils.should_strip_ansi = old_should_strip_ansi
formatting.FORCED_WIDTH = old_forced_width
def setup_locale(locale, localization_proxy=None, text_mode=False):
"""
Procedure setting the system to use the given locale and store it in to the
localization module (if given). DOES NOT PERFORM ANY CHECKS OF THE GIVEN
LOCALE.
$LANG must be set by the caller in order to set the language used by gettext.
Doing this in a thread-safe way is up to the caller.
We also try to set a proper console font for the locale in text mode.
If the font for the locale can't be displayed in the Linux console,
we fall back to the English locale.
This function returns the locale that was used in the setlocale call, which,
depending on what the environment and interface is able to support, may be
different from the locale requested.
:param str locale: locale to setup
:param localization_proxy: DBus proxy of the localization module or None
:param bool text_mode: if the locale is being setup for text mode
:return: the locale that was actually set
:rtype: str
"""
if localization_proxy:
localization_proxy.SetLanguage(locale)
# not all locales might be displayable in text mode
if text_mode:
# check if the script corresponding to the locale/language
# can be displayed by the Linux console
# * all scripts for the given locale/language need to be
# supported by the linux console
# * otherwise users might get a screen full of white rectangles
# (also known as "tofu") in text mode
# then we also need to check if we have information about what
# font to use for correctly displaying the given language/locale
script_supported = locale_supported_in_console(locale)
log.debug("scripts found for locale %s: %s", locale,
get_locale_scripts(locale))
console_fonts = get_locale_console_fonts(locale)
log.debug("console fonts found for locale %s: %s", locale,
console_fonts)
font_set = False
if script_supported and console_fonts:
# try to set console font
for font in console_fonts:
if set_console_font(font):
# console font set successfully, skip the rest
font_set = True
break
if not font_set:
log.warning("can't set console font for locale %s", locale)
# report what exactly went wrong
if not (script_supported):
log.warning("script not supported by console for locale %s",
locale)
if not (console_fonts): # no fonts known for locale
log.warning("no console font found for locale %s", locale)
if script_supported and console_fonts:
log.warning(
"none of the suggested fonts can be set for locale %s",
locale)
log.warning("falling back to the English locale")
locale = constants.DEFAULT_LANG
os.environ["LANG"] = locale # pylint: disable=environment-modify
# set the locale to the value we have selected
# Since glibc does not install all locales, an installable locale may not
# actually be available right now. Give it a shot and fallback.
log.debug("setting locale to: %s", locale)
setenv("LANG", locale)
try:
locale_mod.setlocale(locale_mod.LC_ALL, locale)
except locale_mod.Error as e:
log.debug("setlocale failed: %s", e)
locale = constants.DEFAULT_LANG
setenv("LANG", locale)
locale_mod.setlocale(locale_mod.LC_ALL, locale)
# This part is pretty gross:
# In python3, sys.stdout and friends are TextIOWrapper objects that translate
# betwen str types (in python) and byte types (used to actually read from or
# write to the console). These wrappers are configured at startup using the
# locale at startup, which means that if anaconda starts with LANG=C or LANG
# unset, sys.stdout.encoding will be "ascii". And if the language is then changed,
# because anaconda read the kickstart or parsed the command line or whatever,
# say, to Czech, text mode will crash because it's going to try to print non-ASCII
# characters and sys.stdout doesn't know what to do with them. So, when changing
# the locale, create new objects for the standard streams so they are created with
# the new locale's encoding.
# Replacing stdout is about as stable as it looks, and it seems to break when done
# after the GUI is started. Only make the switch if the encoding actually changed.
if locale_mod.getpreferredencoding() != sys.stdout.encoding:
sys.stdin = io.TextIOWrapper(sys.stdin.detach())
sys.stdout = io.TextIOWrapper(sys.stdout.detach())
sys.stderr = io.TextIOWrapper(sys.stderr.detach())
return locale
def open(self, index, subindex=0, mode="rb", encoding="ascii",
buffering=1024, size=None, block_transfer=False, force_segment=False, request_crc_support=True):
"""Open the data stream as a file like object.
:param int index:
Index of object to open.
:param int subindex:
Sub-index of object to open.
:param str mode:
========= ==========================================================
Character Meaning
--------- ----------------------------------------------------------
'r' open for reading (default)
'w' open for writing
'b' binary mode (default)
't' text mode
========= ==========================================================
:param str encoding:
The str name of the encoding used to decode or encode the file.
This will only be used in text mode.
:param int buffering:
An optional integer used to set the buffering policy. Pass 0 to
switch buffering off (only allowed in binary mode), 1 to select line
buffering (only usable in text mode), and an integer > 1 to indicate
the size in bytes of a fixed-size chunk buffer.
:param int size:
Size of data to that will be transmitted.
:param bool block_transfer:
If block transfer should be used.
:param bool force_segment:
Force use of segmented download regardless of data size.
:param bool request_crc_support:
If crc calculation should be requested when using block transfer
:returns:
A file like object.
"""
buffer_size = buffering if buffering > 1 else io.DEFAULT_BUFFER_SIZE
if "r" in mode:
if block_transfer:
raw_stream = BlockUploadStream(self, index, subindex, request_crc_support=request_crc_support)
else:
raw_stream = ReadableStream(self, index, subindex)
if buffering:
buffered_stream = io.BufferedReader(raw_stream, buffer_size=buffer_size)
else:
return raw_stream
if "w" in mode:
if block_transfer:
raw_stream = BlockDownloadStream(self, index, subindex, size, request_crc_support=request_crc_support)
else:
raw_stream = WritableStream(self, index, subindex, size, force_segment)
if buffering:
buffered_stream = io.BufferedWriter(raw_stream, buffer_size=buffer_size)
else:
return raw_stream
if "b" not in mode:
# Text mode
line_buffering = buffering == 1
return io.TextIOWrapper(buffered_stream, encoding,
line_buffering=line_buffering)
return buffered_stream
def main():
"""The entry point for the program."""
if '--' in sys.argv:
cli_args = sys.argv[1:sys.argv.index('--')]
generator_args = sys.argv[sys.argv.index('--') + 1:]
else:
cli_args = sys.argv[1:]
generator_args = []
args = _cmdline_parser.parse_args(cli_args)
debug = False
if args.verbose is None:
logging_level = logging.WARNING
elif args.verbose == 1:
logging_level = logging.INFO
elif args.verbose == 2:
logging_level = logging.DEBUG
debug = True
else:
print('error: I can only be so garrulous, try -vv.', file=sys.stderr)
sys.exit(1)
logging.basicConfig(level=logging_level)
if args.spec and args.spec[0].startswith('+') and args.spec[0].endswith(
'.py'):
# Hack: Special case for defining a spec in Python for testing purposes
# Use this if you want to define a Stone spec using a Python module.
# The module should should contain an api variable that references a
# :class:`stone.api.Api` object.
try:
api = imp.load_source('api', args.api[0]).api
except ImportError as e:
print('error: Could not import API description due to:',
e,
file=sys.stderr)
sys.exit(1)
else:
if args.spec:
specs = []
read_from_stdin = False
for spec_path in args.spec:
if spec_path == '-':
read_from_stdin = True
elif not spec_path.endswith('.stone'):
print(
"error: Specification '%s' must have a .stone extension."
% spec_path,
file=sys.stderr)
sys.exit(1)
elif not os.path.exists(spec_path):
print("error: Specification '%s' cannot be found." %
spec_path,
file=sys.stderr)
sys.exit(1)
else:
with open(spec_path) as f:
specs.append((spec_path, f.read()))
if read_from_stdin and specs:
print(
"error: Do not specify stdin and specification files "
"simultaneously.",
file=sys.stderr)
sys.exit(1)
if not args.spec or read_from_stdin:
specs = []
if debug:
print('Reading specification from stdin.')
if six.PY2:
UTF8Reader = codecs.getreader('utf8')
sys.stdin = UTF8Reader(sys.stdin)
stdin_text = sys.stdin.read()
else:
stdin_text = io.TextIOWrapper(sys.stdin.buffer,
encoding='utf-8').read()
parts = stdin_text.split('namespace')
if len(parts) == 1:
specs.append(('stdin.1', parts[0]))
else:
specs.append(('stdin.1',
'%snamespace%s' % (parts.pop(0), parts.pop(0))))
while parts:
specs.append(('stdin.%s' % (len(specs) + 1),
'namespace%s' % parts.pop(0)))
if args.filter_by_route_attr:
route_filter, route_filter_errors = parse_route_attr_filter(
args.filter_by_route_attr, debug)
if route_filter_errors:
print('Error(s) in route filter:', file=sys.stderr)
for err in route_filter_errors:
print(err, file=sys.stderr)
sys.exit(1)
else:
route_filter = None
# TODO: Needs version
tower = TowerOfStone(specs, debug=debug)
try:
api = tower.parse()
except InvalidSpec as e:
print('%s:%s: error: %s' % (e.path, e.lineno, e.msg),
file=sys.stderr)
if debug:
print(
'A traceback is included below in case this is a bug in '
'Stone.\n',
traceback.format_exc(),
file=sys.stderr)
sys.exit(1)
if api is None:
print(
'You must fix the above parsing errors for generation to '
'continue.',
file=sys.stderr)
sys.exit(1)
if args.whitelist_namespace_routes:
for namespace_name in args.whitelist_namespace_routes:
if namespace_name not in api.namespaces:
print(
'error: Whitelisted namespace missing from spec: %s' %
namespace_name,
file=sys.stderr)
sys.exit(1)
for namespace in api.namespaces.values():
if namespace.name not in args.whitelist_namespace_routes:
namespace.routes = []
namespace.route_by_name = {}
if args.blacklist_namespace_routes:
for namespace_name in args.blacklist_namespace_routes:
if namespace_name not in api.namespaces:
print(
'error: Blacklisted namespace missing from spec: %s' %
namespace_name,
file=sys.stderr)
sys.exit(1)
else:
api.namespaces[namespace_name].routes = []
api.namespaces[namespace_name].route_by_name = {}
if route_filter:
for namespace in api.namespaces.values():
filtered_routes = []
for route in namespace.routes:
if route_filter.eval(route):
filtered_routes.append(route)
else:
del namespace.route_by_name[route.name]
namespace.routes = filtered_routes
if args.attribute:
attrs = set(args.attribute)
if ':all' in attrs:
attrs = {field.name for field in api.route_schema.fields}
else:
attrs = set()
for namespace in api.namespaces.values():
for route in namespace.routes:
for k in list(route.attrs.keys()):
if k not in attrs:
del route.attrs[k]
# Remove attrs that weren't specified from the route schema
for field in api.route_schema.fields[:]:
if field.name not in attrs:
api.route_schema.fields.remove(field)
del api.route_schema._fields_by_name[field.name]
else:
attrs.remove(field.name)
# Error if specified attr isn't even a field in the route schema
if attrs:
attr = attrs.pop()
print('error: Attribute not defined in stone_cfg.Route: %s' % attr,
file=sys.stderr)
sys.exit(1)
if args.generator in _builtin_generators:
generator_module = __import__('stone.target.%s' % args.generator,
fromlist=[''])
elif not os.path.exists(args.generator):
print("error: Generator '%s' cannot be found." % args.generator,
file=sys.stderr)
sys.exit(1)
elif not os.path.isfile(args.generator):
print("error: Generator '%s' must be a file." % args.generator,
file=sys.stderr)
sys.exit(1)
elif not Compiler.is_stone_generator(args.generator):
print("error: Generator '%s' must have a .stoneg.py extension." %
args.generator,
file=sys.stderr)
sys.exit(1)
else:
# A bit hacky, but we add the folder that the generator is in to our
# python path to support the case where the generator imports other
# files in its local directory.
new_python_path = os.path.dirname(args.generator)
if new_python_path not in sys.path:
sys.path.append(new_python_path)
try:
generator_module = imp.load_source('user_generator',
args.generator)
except:
print(
"error: Importing generator '%s' module raised an exception:" %
args.generator,
file=sys.stderr)
raise
c = Compiler(
api,
generator_module,
generator_args,
args.output,
clean_build=args.clean_build,
)
try:
c.build()
except GeneratorException as e:
print('%s: error: %s raised an exception:\n%s' %
(args.generator, e.generator_name, e.traceback),
file=sys.stderr)
sys.exit(1)
if not sys.argv[0].endswith('stone'):
# If we aren't running from an entry_point, then return api to make it
# easier to do debugging.
return api
#=============================================================================
# Globals
# Python has different string types depending on the python version. We must
# be able to identify them both.
# pylint: disable=undefined-variable
STRTYPES = [str] if sys.version_info > (3, ) else [str, unicode]
# Pandoc uses UTF-8 for both input and output; so must its filters. This is
# handled differently depending on the python version.
if sys.version_info > (3, ):
# Py3 strings are unicode: https://docs.python.org/3.5/howto/unicode.html.
# Character encoding/decoding is performed automatically at stream
# interfaces: https://stackoverflow.com/questions/16549332/.
# Set it to UTF-8 for all streams.
STDIN = io.TextIOWrapper(sys.stdin.buffer, 'utf-8', 'strict')
STDOUT = io.TextIOWrapper(sys.stdout.buffer, 'utf-8', 'strict')
STDERR = io.TextIOWrapper(sys.stderr.buffer, 'utf-8', 'strict')
else:
# Py2 strings are ASCII bytes. Encoding/decoding is handled separately.
# See: https://docs.python.org/2/howto/unicode.html.
STDIN = sys.stdin
STDOUT = sys.stdout
STDERR = sys.stdout
# Privately flags that cleveref TeX needs to be written into the doc
# (for TeX/pdf output only).
_cleveref_tex_flag = False # pylint: disable=invalid-name
# Used to track section numbers
# pylint: disable=invalid-name
