def roundedSummaryStats(self,array):
[mean,stddev,min,max]=SummaryStats.summaryStats(array)
mean=int(100.0*mean+5.0/9.0)/100.0
stddev=int(100.0*stddev+5.0/9.0)/100.0
min=int(100.0*min+5.0/9.0)/100.0
max=int(100.0*max+5.0/9.0)/100.0
return [mean,stddev,min,max]
def readnumber(number):
"""
:param float number: a float number (with decimals) indicating a quantity
:return: a text that indicates the full amount in word form, properly ending each digit with the right term.
"""
position_call = ["แสน", "หมื่น", "พัน", "ร้อย", "สิบ", ""]
number_call = ["", "หนึ่ง", "สอง", "สาม","สี่", "ห้า", "หก", "เจ็ด", "แปด", "เก้า"]
number = number
ret = ""
if (number == 0): return ret
if (number > 1000000):
ret += readnumber(int(number / 1000000)) + "ล้าน"
number = int(math.fmod(number, 1000000))
divider = 100000
pos = 0
while(number > 0):
d=int(number/divider)
if (divider == 10) and (d == 2):
ret += "ยี่"
elif (divider == 10) and (d == 1):
ret += ""
elif ((divider == 1) and (d == 1) and (ret != "")):
ret += "เอ็ด"
else:
ret += number_call[d]
if d:
ret += position_call[pos]
else:
ret += ""
number=number % divider
divider=divider / 10
pos += 1
return ret
def printcoords(event):
zoom = sc.get()
x = int(event.x/zoom)
y = int(event.y/zoom)
selected["x"]=x
selected["y"]=y
updateImage()
def load(name):
fs_name = format.path("{0}.chunks".format(name))
if not os.path.exists(fs_name):
raise IOError
with io.open(fs_name) as fp:
name = fp.readline()[:-1]
size = fp.readline()[:-1]
if name.startswith("name:") and size.startswith("size:"):
name = name[5:]
size = size[5:]
else:
raise TypeError("chunk.file has wrong format")
ci = ChunkInfo(name)
ci.loaded = True
ci.set_size(size)
while True:
if not fp.readline(): #: skip line
break
name = fp.readline()[1:-1]
range = fp.readline()[1:-1]
if name.startswith("name:") and range.startswith("range:"):
name = name[5:]
range = range[6:].split("-")
else:
raise TypeError("chunk.file has wrong format")
ci.add_chunk(name, (int(range[0]), int(range[1])))
return ci
def updateImage():
if selected["err"]:
selected["err"].destroy()
selected["err"]=None
zoom = sc.get()
newImg = originalImg.convert("RGB")
for nt, x, y in selected["triples"]:
try: #1.1.6 and above
newImg[x,y] = (0,150,255*nt//len(cg.seq))
except:
newImg.putpixel((x,y), (0,150,255*nt//len(cg.seq)))
if selected["x"] and selected["y"]:
x,y = selected["x"], selected["y"]
try: #1.1.6 and above
newImg[x,y] = (255,0,0)
except:
newImg.putpixel((x,y), (255,0,0))
w = tk.Label(root, text="Selected coordinates: {},{}".format(x,y))
w.grid(row = 4, column = 0, sticky = "E")
w = tk.Label(root, text="Please choose corresponding nucleotide\n(1-based coordinates):")
w.grid(row = 5, column = 0, sticky = "E")
w = tk.Entry(root)
selected["nt_entry"] = w
w.grid(row = 5, column = 1, sticky = "W")
w.focus_set()
w = tk.Button(root, text="OK", command=submitSelected)
w.grid(row = 5, column = 1)
newsize = (originalImg.size[0]*int(zoom), originalImg.size[1]*int(zoom))
newImg = newImg.resize(newsize, Image.NEAREST)
img = ImageTk.PhotoImage(newImg)
imgDisplay.configure(image = img)
imgDisplay.image = img
def get_partners_huwknd(op_number):
"""
List of partner mentioning of OP in Huwknd table
"""
print("get_partners_huwknd()")
npartners = 0
partners = []
huwknd_qs = get_huwknd(op_number)
if huwknd_qs is None:
print("Huwknd does not contain partners for OP %d", op_number)
else:
nhuwknds = huwknd_qs.count()
print("Huwknd has %d entries for OP %s" % (nhuwknds, op_number))
for huwknd in huwknd_qs:
mar_date = "%02d/%02d/%04d" % (int(huwknd.hdag), int(huwknd.hmaand), int(huwknd.hjaar))
gebsex = huwknd.gebsex
if gebsex == 'm': # OP is man
partnersex = 'v'
familyname = huwknd.anmhv # familyname wife
firstname1 = huwknd.vrn1hv # firstname1 wife
firstname2 = huwknd.vrn2hv # firstname2 wife
firstname3 = huwknd.vrn3hv # firstname3 wife
elif gebsex == 'v': # OP is woman
partnersex = 'm'
familyname = huwknd.anmhm # familyname husband
firstname1 = huwknd.vrn1hm # firstname1 husband
firstname2 = huwknd.vrn2hm # firstname2 husband
firstname3 = huwknd.vrn3hm # firstname3 husband
if familyname is None: familyname = ""
if firstname1 is None: firstname1 = ""
if firstname2 is None: firstname2 = ""
if firstname3 is None: firstname3 = ""
fullname = familyname
fullname += ", "
fullname += firstname1
if firstname2 != "":
fullname += " "
fullname += firstname2
if firstname3 != "":
fullname += " "
fullname += firstname3
if gebsex == 'm' or gebsex == 'v':
partner = {
"mar_date": mar_date,
"sex": partnersex,
"fullname": fullname
}
npartners += 1
print("partner:", partner)
partners.append(partner)
print("Huwknd has %d partner entries for OP %s" % (npartners, op_number))
return partners
def __init__(self, delimiter=None, comments='#', autostrip=True, encoding=None):
delimiter = _decode_line(delimiter)
comments = _decode_line(comments)
self.comments = comments
# Delimiter is a character
if (delimiter is None) or isinstance(delimiter, basestring):
delimiter = delimiter or None
_handyman = self._delimited_splitter
# Delimiter is a list of field widths
elif hasattr(delimiter, '__iter__'):
_handyman = self._variablewidth_splitter
idx = np.cumsum([0] + list(delimiter))
delimiter = [slice(i, j) for (i, j) in zip(idx[:-1], idx[1:])]
# Delimiter is a single integer
elif int(delimiter):
(_handyman, delimiter) = (
self._fixedwidth_splitter, int(delimiter))
else:
(_handyman, delimiter) = (self._delimited_splitter, None)
self.delimiter = delimiter
if autostrip:
self._handyman = self.autostrip(_handyman)
else:
self._handyman = _handyman
self.encoding = encoding
def Waveform(addr, count, isTA, write=False, label=None): header = (WFM << 4) | (write & 0x1) count = int(count) count = ((count // ADDRESS_UNIT)-1) & 0x000fffff # 20 bit count addr = (addr // ADDRESS_UNIT) & 0x00ffffff # 24 bit addr payload = (PLAY << WFM_OP_OFFSET) | ((int(isTA) & 0x1) << TA_PAIR_BIT) | (count << 24) | addr return Instruction(header, payload, label)
def _parse_response(response):
tokens = ['']
fresh = True
quoted = False
just_unquoted = False
for c in response:
if c == '"':
if just_unquoted:
tokens[-1] += '"' # False alarm, this is an escaped ".
fresh = False
just_unquoted = quoted
quoted = not quoted
elif c == ' ' and not quoted:
try:
tokens[-1] = int(tokens[-1], 0)
except ValueError:
pass
tokens.append('')
fresh = True
just_unquoted = False
else:
tokens[-1] += c
fresh = False
just_unquoted = False
try:
tokens[-1] = int(tokens[-1], 0)
except ValueError:
pass
if fresh:
tokens.pop()
return tokens
def test_typed_set(self):
my_set = caom_util.TypedSet(str, )
with self.assertRaises(AssertionError):
my_set.add(float(1.0))
my_set.add(int(1))
my_set.add(bool(1))
self.assertRaises(AssertionError, caom_util.TypedSet, str, float(1.0))
my_set = caom_util.TypedSet(str, "Test1")
self.assertEqual(1, len(my_set))
self.assertEqual("Test1", my_set.pop())
my_set.add("Test1")
my_set.add("Test2")
self.assertEqual(2, len(my_set), "set length")
with self.assertRaises(AssertionError):
my_set.add(float(1.0))
my_set.add(int(1))
my_set.add(bool(1))
my_set.add("Test1")
my_set = caom_util.TypedSet(str, )
my_set.add("Test1")
my_set.add("Test1")
self.assertTrue(len(my_set) == 1)
def find_new_gid(sysuser, preferred_gid=None):
defs = get_defs()
# TODO: Catch errors
if not sysuser:
gid_min = int(defs.get("GID_MIN", 1000))
gid_max = int(defs.get("GID_MAX", 60000))
else:
gid_min = int(defs.get("SYS_GID_MIN", 101))
gid_max = int(defs.get("GID_MIN", 1000))
gid_max = int(defs.get("SYS_GID_MAX", gid_max))
if preferred_gid and gid_min < preferred_gid < gid_max:
try:
grp.getgrgid(preferred_gid)
except KeyError:
return preferred_gid
if sysuser:
for uid in range(gid_max, gid_min, -1):
try:
grp.getgrgid(uid)
except KeyError:
return uid
else:
for uid in range(gid_min, gid_max):
try:
grp.getgrgid(uid)
except KeyError:
return uid
syslog.syslog(syslog.LOG_WARNING, "no more available GID on the system")
def register_tasks(namespaces, default_namespace={}, dry_run_suffix=''):
"""Return a Luigi task class after parsed Luigi task metadata.
:param dict namespaces: Task namespaces.
:param dict default_namespace: Default namespaces.
:param unicode dry_run_suffix: Suffix to be added to file created during dry run.
:rtype: iterable
"""
for task_name, namespace in namespaces.items():
action_namespace = default_namespace.copy()
action_namespace.update(namespace)
task_keys = ['target_pattern', 'sources_repls', 'action_template', 'SHELL']
task_namespace = {k: action_namespace[k] for k in task_keys if k in action_namespace}
task_namespace['sources_repls'] = task_namespace['sources_repls'].split()
# luigi attributes
task_namespace['resources'] = {k.partition('_')[2]: int(v) for k, v in namespace.items()
if k.startswith('RESOURCES_')}
task_namespace.update(
{k: int(namespace[k]) for k in ['priority', 'disabled', 'worker_timeout']
if k in namespace})
yield ReRuleTask.factory(
task_name, dry_run_suffix=dry_run_suffix, action_namespace=action_namespace,
**task_namespace
)
def parse_date(self, agetd, entry):
m = self.age_pattern.search(agetd.text)
days = None
hours = None
if m:
days = int(m.group("days1"))
hours = int(m.group("days2")) * 2.4
else:
p = re.compile(r"(?P<hours>\d+) hours?")
m = p.search(agetd.text)
if m:
days = 0
hours = int(m.group("hours"))
if hours is not None:
pubdate = arrow.utcnow().replace(days=-days, hours=-1) # hours because of timezone change below
if hours > 0:
pubdate = pubdate.replace(hours=-hours)
pubdate = pubdate.to("+01:00") # nzbindex server time, I guess?
entry.epoch = pubdate.timestamp
entry.pubdate_utc = str(pubdate)
entry.age_days = (arrow.utcnow() - pubdate).days
entry.age = str(entry.age_days) + "d"
entry.age_precise = True # Precise to 2.4 hours, should be enough for duplicate detection
entry.pubDate = pubdate.format("ddd, DD MMM YYYY HH:mm:ss Z")
else:
self.error("Found no age info in %s" % str(agetd))
raise IndexerResultParsingRowException("Unable to parse age")
def test_init(self):
dimension = wcs.Dimension2D(int(1), int(2))
ref_coord = wcs.Coord2D(wcs.RefCoord(float(9.0), float(10.0)),
wcs.RefCoord(float(11.0), float(12.0)))
cd11 = float(1.1)
cd12 = float(1.2)
cd21 = float(2.1)
cd22 = float(2.2)
self.assertRaises(TypeError, wcs.CoordFunction2D, None,
ref_coord, cd11, cd12, cd21, cd22)
self.assertRaises(TypeError, wcs.CoordFunction2D, dimension,
None, cd11, cd12, cd21, cd22)
self.assertRaises(TypeError, wcs.CoordFunction2D, dimension,
ref_coord, None, cd12, cd21, cd22)
self.assertRaises(TypeError, wcs.CoordFunction2D, dimension,
ref_coord, cd11, None, cd21, cd22)
self.assertRaises(TypeError, wcs.CoordFunction2D, dimension,
ref_coord, cd11, cd12, None, cd22)
self.assertRaises(TypeError, wcs.CoordFunction2D, dimension,
ref_coord, cd11, cd12, cd21, None)
function = wcs.CoordFunction2D(dimension, ref_coord,
cd11, cd12, cd21, cd22)
self.assertEqual(function.dimension, dimension)
self.assertEqual(function.ref_coord, ref_coord)
self.assertEqual(function.cd11, cd11)
self.assertEqual(function.cd12, cd12)
self.assertEqual(function.cd21, cd21)
self.assertEqual(function.cd22, cd22)
def initializeVisibilityMatrix(self):
global VISIBILITY_MATRIX_CACHE
if reduce(str.__add__, self.layoutText) not in VISIBILITY_MATRIX_CACHE:
from .game import Directions
vecs = [(-0.5, 0), (0.5, 0), (0, -0.5), (0, 0.5)]
dirs = [
Directions.NORTH,
Directions.SOUTH,
Directions.WEST,
Directions.EAST]
vis = Grid(
self.width,
self.height,
{Directions.NORTH: set(),
Directions.SOUTH: set(),
Directions.EAST: set(),
Directions.WEST: set(),
Directions.STOP: set()})
for x in range(self.width):
for y in range(self.height):
if self.walls[x][y] == False:
for vec, direction in zip(vecs, dirs):
dx, dy = vec
nextx, nexty = x + dx, y + dy
while (nextx + nexty) != int(nextx) + int(nexty) or not self.walls[int(nextx)][int(nexty)]:
vis[x][y][direction].add((nextx, nexty))
nextx, nexty = x + dx, y + dy
self.visibility = vis
VISIBILITY_MATRIX_CACHE[reduce(str.__add__, self.layoutText)] = vis
else:
self.visibility = VISIBILITY_MATRIX_CACHE[
reduce(str.__add__, self.layoutText)]
def pre_process (self, rec):
"""
Clean flanking whitespace, ensure rec has all fields and parse out structured new fields.
"""
# make sure that every field in the record & strip flanking wspace
for f in consts.ALL_NAMES:
if f in rec.keys():
rec[f] = str (rec[f]).strip()
else:
rec[f] = ''
# now parse out the structured / metadata fields
try:
rec['tags'] = self.parse_tags_str (rec.get ('tags', ''))
rec['repeat'] = self.parse_repeat_str (rec.get ('repeat', ''))
return rec
except:
print ("Rec %s has problems with its repeat or tag fields" % rec[Column.variable.value])
raise
if rec[Column.text_validation_type.value] == 'integer':
min = rec[Column.text_validation_min.value]
if min:
rec[Column.text_validation_min.value] = str ("%d" % int (min))
max = rec[Column.text_validation_max.value]
if max:
rec[Column.text_validation_max.value] = str ("%d" % int (max))
def test_init(self):
self.assertRaises(TypeError, wcs.CoordAxis1D, None)
self.assertRaises(TypeError, wcs.CoordAxis1D, int(1))
axis = wcs.Axis("ctype", "cunit")
axis_1d = wcs.CoordAxis1D(axis)
self.assertEqual(axis_1d.axis, axis)
with self.assertRaises(TypeError):
axis_1d.error = str("s")
axis_1d.bounds = str("s")
axis_1d.function = str("s")
axis_1d.range = str("s")
error = wcs.CoordError(float(1.0), float(2.0))
axis_1d.error = error
self.assertEqual(axis_1d.error, error)
start = wcs.RefCoord(float(1.0), float(2.0))
end = wcs.RefCoord(float(3.0), float(4.0))
coord_range = wcs.CoordRange1D(start, end)
axis_1d.range = coord_range
self.assertEqual(axis_1d.range, coord_range)
bounds = wcs.CoordBounds1D()
axis_1d.bounds = bounds
self.assertEqual(axis_1d.bounds, bounds)
naxis = int(1)
delta = float(2.5)
ref_coord = wcs.RefCoord(float(1.0), float(2.0))
function = wcs.CoordFunction1D(naxis, delta, ref_coord)
axis_1d.function = function
self.assertEqual(axis_1d.function, function)
def write_entry(self, key, value, encoding):
if encoding == "hex2bin":
if len(value) % 2 != 0:
raise InputError("%s: Invalid data length. Should be multiple of 2." % key)
value = binascii.a2b_hex(value)
if encoding == "base64":
value = binascii.a2b_base64(value)
if encoding == "string":
if type(value) == bytes:
value = value.decode()
value += '\0'
encoding = encoding.lower()
varlen_encodings = ["string", "binary", "hex2bin", "base64"]
primitive_encodings = ["u8", "i8", "u16", "u32", "i32"]
if encoding in varlen_encodings:
try:
self.cur_page.write_varlen_data(key, value, encoding, self.namespace_idx,self)
except PageFullError:
new_page = self.create_new_page()
new_page.write_varlen_data(key, value, encoding, self.namespace_idx,self)
elif encoding in primitive_encodings:
try:
self.cur_page.write_primitive_data(key, int(value), encoding, self.namespace_idx,self)
except PageFullError:
new_page = self.create_new_page()
new_page.write_primitive_data(key, int(value), encoding, self.namespace_idx,self)
else:
raise InputError("%s: Unsupported encoding" % encoding)
def makeTranscript(self,root):
id=root["extra"]["ID"]
strand=root["strand"]
transcript=Transcript(id,strand)
root["object"]=transcript
transcript.substrate=root["substrate"]
transcript.source=root["source"]
transcript.begin=int(root["begin"])
transcript.end=int(root["end"])
children=root.get("children",None)
if(children is None): return transcript
for child in children:
obj=self.labelStructure(child)
if(obj is None): continue
if(type(obj)==Exon):
childType=child["type"]
if(childType=="CDS" or
re.search("-exon",childType)): transcript.addExon(obj)
elif(childType=="exon"): transcript.addRawExon(obj)
elif(re.search("UTR",childType)): transcript.addUTR(obj)
obj.transcript=transcript
transcript.parseRawExons()
transcript.setExonTypes()
transcript.setUTRtypes()
transcript.sortExons()
transcript.adjustOrders()
extra=root["extra"]
transcript.extraFields=""
for key in extra:
transcript.extraFields+=key+"="+extra[key]+";"
return transcript
def loadCounts(indiv,hap,filename,changes,hash):
transcripts=changes.keys()
for transcript in transcripts:
if(changes[transcript]=="cryptic-site"):
if(not rex.find("ALT\d+_(\S+)_(\d+)",transcript)):
raise Exception(transcript)
if(int(rex[2])!=hap): continue
baseTrans=rex[1]+"_"+rex[2]
if(not expressed.get(rex[1],False)): continue
rec=hash.get(indiv+baseTrans,None)
if(not rec):
rec=hash[indiv+baseTrans]={"numSites":0}
rec["numSites"]+=1
with open(filename,"rt") as fh:
for line in fh:
fields=line.split()
if(len(fields)!=2): continue
(altID,count)=fields
if(not changes.get(altID,None)): continue
type=changes[altID]
if(type!="cryptic-site"): continue
if(not rex.find("ALT\d+_(\S+)_(\d+)",altID)):
raise Exception(baseTrans)
baseTrans=rex[1]+"_"+rex[2]
if(hash[indiv+baseTrans].get("supported",None) is None):
hash[indiv+baseTrans]["supported"]=0
if(int(count)>=MIN_COUNT):
hash[indiv+baseTrans]["supported"]+=1
def _get_terminal_size_linux():
def ioctl_GWINSZ(fd):
try:
import fcntl
import termios
cr = struct.unpack('hh',
fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
return cr
except Exception as e:
log.error(e)
pass
cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
if not cr:
try:
fd = os.open(os.ctermid(), os.O_RDONLY)
cr = ioctl_GWINSZ(fd)
os.close(fd)
except Exception as e:
log.error(e)
pass
if not cr:
try:
cr = (os.environ['LINES'], os.environ['COLUMNS'])
except Exception as e:
log.error(e)
return None
return int(cr[1]), int(cr[0])
def _parse_percents(male_percent_str, female_percent_str):
if not (male_percent_str or female_percent_str):
# Neither specified. 50% each.
male_percent = 50
female_percent = 50
else:
r = re.compile('^\d+$')
male_percent = None
female_percent = None
if male_percent_str:
if not r.match(male_percent_str):
_die_usage('Bad male percent: {0}'.format(male_percent_str))
male_percent = int(male_percent_str)
if female_percent_str:
if not r.match(female_percent_str):
_die_usage('Bad female percent: {0}'.format(female_percent_str))
female_percent = int(female_percent_str)
if not female_percent:
# We have a male value, but not a female one.
female_percent = 100 - male_percent
if not male_percent:
# We have a male value, but not a female one.
male_percent = 100 - female_percent
if (male_percent + female_percent) != 100:
_die_usage("Male and female percentages don't add up to 100.")
return (male_percent, female_percent)
def update_usage(self, line=None):
"""
Check usage percentage for this mountpoint.
Returns dictionary with usage details.
"""
if self.filesystem in PSEUDO_FILESYSTEMS:
return {}
if line is None:
parser = ShellCommandParser()
try:
stdout, stderr = parser.execute('df', '-Pk', self.mountpoint)
except ShellCommandParserError:
raise FileSystemError('Error getting usage for {0}'.format(self.mountpoint))
header, usage = stdout.split('\n', 1)
try:
usage = ' '.join(usage.split('\n'))
except ValueError:
pass
else:
usage = ' '.join(line.split('\n'))
fs, size, used, free, percent, mp = [x.strip() for x in usage.split()]
percent = percent.rstrip('%')
self.usage = {
'mountpoint': self.mountpoint,
'size': int(size),
'used': int(used),
'free': int(free),
'percent': int(percent),
}
def get_package_stats(self, pid=None, root=None, owner=None):
qry = ("SELECT p.pid, SUM(f.size) AS sizetotal, COUNT(f.fid) AS linkstotal, sizedone, linksdone "
"FROM packages p JOIN files f ON p.pid = f.package AND f.dlstatus > 0 {0} LEFT OUTER JOIN "
"(SELECT p.pid AS pid, SUM(f.size) AS sizedone, COUNT(f.fid) AS linksdone "
"FROM packages p JOIN files f ON p.pid = f.package {0} AND f.dlstatus in (5,6) GROUP BY p.pid) s ON s.pid = p.pid "
"GROUP BY p.pid")
# status in (finished, skipped, processing)
if root is not None:
self.c.execute(qry.format(
"AND (p.root=:root OR p.pid=:root)"), locals())
elif pid is not None:
self.c.execute(qry.format("AND p.pid=:pid"), locals())
elif owner is not None:
self.c.execute(qry.format("AND p.owner=:owner"), locals())
else:
self.c.execute(qry.format(""))
data = {}
for r in self.c.fetchall():
data[r[0]] = PackageStats(
r[2] if r[2] else 0,
r[4] if r[4] else 0,
int(r[1]) if r[1] else 0,
int(r[3]) if r[3] else 0,
)
return data
def write_data(self, data, dstart=None, reshape_order='C'):
"""Write ``data`` to `file`.
Parameters
----------
data : `array-like`
Data that should be written to `file`.
dstart : non-negative int, optional
Offset in bytes of the start position of the written data.
By default, it is taken to be `header_size`.
reshape_order : {'C', 'F', 'A'}, optional
Value passed as ``order`` parameter to `numpy.reshape`.
Reshaping is only done in case the whole data block is read.
See Also
--------
write_header
"""
data = np.asarray(data).reshape(-1, order=reshape_order)
if dstart is None:
dstart = int(self.header_size)
elif dstart < 0:
raise ValueError('`dstart` must be non-negative, got {}'
''.format(dstart))
else:
dstart = int(dstart)
if dstart < self.header_size:
raise ValueError('invalid `dstart`, resulting in absolute '
'`dstart` < `header_size` ({} < {})'
''.format(dstart, self.header_size))
self.file.seek(dstart)
data.tofile(self.file)
def update_usage(self, line=None):
"""
Check usage percentage for this mountpoint.
Returns dictionary with usage details.
"""
if line is None:
parser = ShellCommandParser()
try:
stdout, stderr = parser.execute('df', '-k', self.mountpoint)
except ShellCommandParserError as e:
raise FileSystemError('Error checking filesystem usage: {0}'.format(e))
header, usage = stdout.split('\n', 1)
else:
usage = line
m = RE_DF.match(usage)
if not m:
raise FileSystemError('Error matching df output line: {0}'.format(usage))
self.usage = {
'mountpoint': self.mountpoint,
'size': int(m.group(2)),
'used': int(m.group(3)),
'free': int(m.group(4)),
'percent': int(m.group(5))
}
def getFeatures(self, state, action):
# extract the grid of food and wall locations and get the ghost
# locations
food = state.getFood()
walls = state.getWalls()
ghosts = state.getGhostPositions()
features = util.Counter()
features["bias"] = 1.0
# compute the location of pacman after he takes the action
x, y = state.getPacmanPosition()
dx, dy = Actions.directionToVector(action)
next_x, next_y = int(x + dx), int(y + dy)
# count the number of ghosts 1-step away
features["#-of-ghosts-1-step-away"] = sum((next_x, next_y) in Actions.getLegalNeighbors(g, walls)
for g in ghosts)
# if there is no danger of ghosts then add the food feature
if not features["#-of-ghosts-1-step-away"] and food[next_x][next_y]:
features["eats-food"] = 1.0
dist = closestFood((next_x, next_y), food, walls)
if dist is not None:
# make the distance a number less than one otherwise the update
# will diverge wildly
features["closest-food"] = old_div(float(dist), \
(walls.width * walls.height))
features.divideAll(10.0)
return features
def submitted_jobs_user(username=None, path=None):
"""
returns info on submitted jobs for given user
as a list of dictionaries
"""
if username is None:
username = get_current_user()
output = subprocess.Popen(['condor_q', username, "-long"],
stdout=subprocess.PIPE).communicate()[0]
jobs = output.strip().split("\n\n")
joblist = []
for job in jobs:
j = re.findall("ClusterId = (.*)\n", job)
if len(j) == 0 and len(jobs) <= 1:
return []
job_id = j[0]
directory = re.findall('Iwd = "(.*)"\n', job)[0]
run_id = int(re.findall('UserLog = ".*/([0-9]*).log"\n', job)[0])
status = int(re.findall('\nJobStatus = ([0-9]*)\n', job)[0])
if path is not None and not directory.startswith(path):
continue
joblist.append(
dict(job_exp_id=job_id,
directory=directory,
run_exp_id=run_id,
status=status))
return joblist
def loadGFF_transcript(self,fields,line,transcriptBeginEnd,GFF,
transcripts,readOrder,genes):
begin=int(fields[3])-1
end=int(fields[4])
rex=Rex()
if(rex.find('transcript_id[:=]?\s*"?([^\s";]+)"?',line)):
transcriptId=rex[1]
transcriptBeginEnd[transcriptId]=[begin,end]
strand=fields[6]
transcriptExtraFields=""
for i in range(8,len(fields)):
transcriptExtraFields+=fields[i]+" "
transcript=transcripts.get(transcriptId,None)
if(transcript is None):
transcripts[transcriptId]=transcript= \
Transcript(transcriptId,strand)
transcript.setStopCodons(self.stopCodons)
transcript.readOrder=readOrder;
readOrder+=1
transcript.substrate=fields[0]
transcript.source=fields[1]
transcript.setBegin(begin)
transcript.setEnd(end)
geneId=None
if(rex.find("genegrp=(\S+)",line)): geneId=rex[1]
elif(rex.find('gene_id[:=]?\s*\"?([^\s\;"]+)\"?',line)):
geneId=rex[1]
if(not geneId): raise Exception("can't parse GTF: "+line)
transcript.geneId=geneId
gene=genes.get(geneId,None)
if(not gene): genes[geneId]=gene=Gene(); gene.setId(geneId)
transcript.setGene(gene)
gene.addTranscript(transcript)
transcript.extraFields=transcriptExtraFields
def addThumbnailBase64():
images = {}
delete_all = False
if delete_all:
for d in getCollDrops().find({"profile_image_url_big_base64":{'$exists':True}}):
del d["profile_image_url_big_base64"]
del d["profile_image_url_small_base64"]
getCollDrops().save(d, safe=True)
cnt = getCollDrops().find({"profile_image_url_big_base64":{'$exists':False}}).count()
print("numdrops:",cnt)
clog("making cache")
for d in getCollDrops().find({"profile_image_url_big_base64":{'$exists':True}}):
bt = d["user"]["profile_image_url"].replace("_normal", "_reasonably_small")
images[bt] = (d["profile_image_url_big_base64"], d["profile_image_url_small_base64"])
clog("making thumbnails")
for d in getCollDrops().find({"profile_image_url_big_base64":{'$exists':False}}):
bt = d["user"]["profile_image_url"].replace("_normal", "_reasonably_small")
cnt -= 1;
if cnt%50==0:
print(cnt, bt)
if bt not in images:
try:
data = urllib.request.urlopen(bt).read()
fdata = cStringIO.StringIO(data)
i = Image.open(fdata)
except (Exception) as e:
print(e)
data = urllib.request.urlopen(d["user"]["profile_image_url"]).read()
fdata = cStringIO.StringIO(data)
i = Image.open(fdata)
if old_div(float(i.size[1]), float(i.size[0]))>0.5:
if cnt%10==0:
print(d["screen_name"], bt)
nh = float(i.size[1]) / float(i.size[0]) * 80
i = i.resize((80, int(nh)),Image.ANTIALIAS)
outdata = cStringIO.StringIO()
bts = bt.split(".")
format = bts[len(bts)-1].lower()
if format=="jpg":
format = "jpeg"
if format!="jpeg" and format!="png" and format!="gif":
pass
else:
i.save(outdata, format)
d["profile_image_url_big_base64"] = "data:image/"+format+";base64,"+base64.encodestring(outdata.getvalue())
nh = float(i.size[1]) / float(i.size[0]) * 30
i = i.resize((30, int(nh)),Image.ANTIALIAS)
outdata = cStringIO.StringIO()
i.save(outdata, format)
d["profile_image_url_small_base64"] = "data:image/"+format+";base64,"+base64.encodestring(outdata.getvalue())
images[bt] = (d["profile_image_url_big_base64"], d["profile_image_url_small_base64"])
else:
print("skipping:", old_div(float(i.size[1]), float(i.size[0])), bt)
else:
if cnt%10==0:
print(d["screen_name"])
d["profile_image_url_big_base64"] = images[bt][0]
d["profile_image_url_small_base64"] = images[bt][1]
getCollDrops().save(d, safe=True)
def _multinode_transfer(method, dest, source, dst, username, ssh_private_key,
rls, mpt):
# type: (str, DestinationSettings, SourceSettings, str, str,
# pathlib.Path, dict, int) -> None
"""Transfer data to multiple destination nodes simultaneously
:param str method: transfer method
:param DestinationSettings dest: destination settings
:param SourceSettings source: source settings
:param str dst: destination path
:param str username: username
:param pathlib.Path: ssh private key
:param dict rls: remote login settings
:param int mpt: max parallel transfers per node
"""
src = source.path
src_incl = source.include
src_excl = source.exclude
psrc = pathlib.Path(src)
# if source isn't a directory, convert it using src_incl
if not psrc.is_dir():
src_excl = None
src_incl = [src]
src = str(psrc.parent)
psrc = psrc.parent
# if split is specified, force to multinode_scp
if (dest.data_transfer.split_files_megabytes is not None
and method != 'multinode_scp'):
logger.warning('forcing transfer method to multinode_scp with split')
method = 'multinode_scp'
buckets = {}
files = {}
rcodes = {}
spfiles = []
spfiles_count = {}
spfiles_count_lock = threading.Lock()
for rkey in rls:
buckets[rkey] = 0
files[rkey] = []
rcodes[rkey] = None
# walk the directory structure
# 1. construct a set of dirs to create on the remote side
# 2. binpack files to different nodes
total_files = 0
dirs = set()
if dest.relative_destination_path is not None:
dirs.add(dest.relative_destination_path)
for entry in util.scantree(src):
rel = pathlib.Path(entry.path).relative_to(psrc)
sparent = str(pathlib.Path(entry.path).relative_to(psrc).parent)
if entry.is_file():
srel = str(rel)
# check filters
if src_excl is not None:
inc = not any([fnmatch.fnmatch(srel, x) for x in src_excl])
else:
inc = True
if src_incl is not None:
inc = any([fnmatch.fnmatch(srel, x) for x in src_incl])
if not inc:
logger.debug('skipping file {} due to filters'.format(
entry.path))
continue
if dest.relative_destination_path is None:
dstpath = '{}{}'.format(dst, rel)
else:
dstpath = '{}{}/{}'.format(dst, dest.relative_destination_path,
rel)
# get key of min bucket values
fsize = entry.stat().st_size
if (dest.data_transfer.split_files_megabytes is not None
and fsize > dest.data_transfer.split_files_megabytes):
nsplits = int(
math.ceil(fsize /
dest.data_transfer.split_files_megabytes))
lpad = int(math.log10(nsplits)) + 1
spfiles.append(dstpath)
spfiles_count[dstpath] = nsplits
n = 0
curr = 0
while True:
end = curr + dest.data_transfer.split_files_megabytes
if end > fsize:
end = fsize
key = min(buckets, key=buckets.get)
buckets[key] += (end - curr)
if n == 0:
dstfname = dstpath
else:
dstfname = '{}.{}{}'.format(dstpath,
_FILE_SPLIT_PREFIX,
str(n).zfill(lpad))
files[key].append((entry.path, dstfname, curr, end))
if end == fsize:
break
curr = end
n += 1
else:
key = min(buckets, key=buckets.get)
buckets[key] += fsize
files[key].append((entry.path, dstpath, None, None))
total_files += 1
# add directory to create
if sparent != '.':
if dest.relative_destination_path is None:
dirs.add(sparent)
else:
dirs.add('{}/{}'.format(dest.relative_destination_path,
sparent))
total_size = sum(buckets.values())
if total_files == 0:
logger.error('no files to ingress')
return
# create remote directories via ssh
if len(dirs) == 0:
logger.debug('no remote directories to create')
else:
logger.debug('creating remote directories: {}'.format(dirs))
dirs = ['mkdir -p {}'.format(x) for x in list(dirs)]
dirs.insert(0, 'cd {}'.format(dst))
_rls = next(iter(rls.values()))
ip = _rls.remote_login_ip_address
port = _rls.remote_login_port
del _rls
mkdircmd = ('ssh -T -x -o StrictHostKeyChecking=no '
'-o UserKnownHostsFile={} -i {} -p {} {}@{} {}'.format(
os.devnull, ssh_private_key, port, username, ip,
util.wrap_commands_in_shell(dirs)))
rc = util.subprocess_with_output(mkdircmd,
shell=True,
suppress_output=True)
if rc == 0:
logger.info('remote directories created on {}'.format(dst))
else:
logger.error('remote directory creation failed')
return
del ip
del port
logger.info(
'ingress data: {0:.4f} MiB in {1} files to transfer, using {2} max '
'parallel transfers per node'.format(total_size / _MEGABYTE,
total_files, mpt))
logger.info('begin ingressing data from {} to {}'.format(src, dst))
nodekeys = list(buckets.keys())
threads = []
start = datetime.datetime.now()
for i in range(0, len(buckets)):
nkey = nodekeys[i]
thr = threading.Thread(
target=_multinode_thread_worker,
args=(method, mpt, nkey, rcodes, files[nkey], spfiles_count,
spfiles_count_lock, rls[nkey].remote_login_ip_address,
rls[nkey].remote_login_port, username, ssh_private_key,
dest.data_transfer.scp_ssh_extra_options,
dest.data_transfer.rsync_extra_options))
threads.append(thr)
thr.start()
for i in range(0, len(buckets)):
threads[i].join()
diff = datetime.datetime.now() - start
del threads
success = True
for nkey in rcodes:
if rcodes[nkey] != 0:
logger.error('data ingress failed to node: {}'.format(nkey))
success = False
if success:
logger.info(
'finished ingressing {0:.4f} MB of data in {1} files from {2} to '
'{3} in {4:.2f} sec ({5:.3f} Mbit/s)'.format(
total_size / _MEGABYTE, total_files, src, dst,
diff.total_seconds(),
(total_size * 8 / 1e6) / diff.total_seconds()))
path = r'C:\Users\nailt\Desktop\data-mining-git\table.xlsx'
file_read = open(path,'r')
excel_data = file_read.readlines
wb = xlrd.open_workbook(path)
sheet_names = wb.sheet_names()
for sheet in wb.sheets():
nrows = sheet.nrows
ncols = sheet.ncols
dataframe = []
for row in range(nrows):
row_carrier = []
for col in range(ncols):
value = sheet.cell(row,col).value
try:
value = str(int(value))
except ValueError:
pass
finally:
row_carrier.append(value)
dataframe.append(row_carrier)
adres_list = []
for i in range(1,nrows-1):
if dataframe[i][0] != dataframe[i+1][0]:
adres_list.append(dataframe[i][0])
else:
pass
dataframe[1]
df = pd.DataFrame(adres_list)
df.to_csv('adres_list.csv',index = False)
def train(hyp, opt, device):
save_dir, epochs, batch_size, total_batch_size, weights, rank = \
Path(opt.save_dir), opt.epochs, opt.batch_size, opt.total_batch_size, opt.weights, opt.global_rank
do_semi = opt.do_semi
# Directories
wdir = save_dir / 'weights'
wdir.mkdir(parents=True, exist_ok=True)
last = wdir / 'last.pt'
best = wdir / 'best.pt'
results_file = save_dir / 'results.txt'
# Save run settings
with open(save_dir / 'hyp.yaml', 'w') as f:
yaml.dump(hyp, f, sort_keys=False)
with open(save_dir / 'opt.yaml', 'w') as f:
yaml.dump(vars(opt), f, sort_keys=False)
# Configure
plots = not opt.evolve #create plots
cuda = device.type != 'cpu'
init_seeds(2 + rank)
with open(opt.data) as f:
data_dict = yaml.load(f, Loader=yaml.SafeLoader)
nc = 1 if opt.single_cls else int(data_dict['nc']) #number of classes
names = ['item'] if opt.single_cls and len(
data_dict['names']) != 1 else data_dict['names']
assert len(names) == nc, '%g names found for nc=%g dataset in %s' % (
len(names), nc, opt.data)
# Model
pretrained = weights.endswith('.pt')
if pretrained:
with torch_distributed_zero_first(rank):
attempt_download(weights) # download if not found locally
ckpt = torch.load(weights, map_location=device) #load checkpoint
model = Model(opt.cfg or ckpt['model'].yaml,
ch=3,
nc=nc,
anchors=hyp.get('anchors')).to(device) #create
exclude = [
'anchor'
] if (opt.cfg or hyp.get('anchors')) and not opt.resume else [
] #exclude keys
state_dict = ckpt['model'].float().state_dict() # to FP32
state_dict = intersect_dicts(state_dict,
model.state_dict(),
exclude=exclude) #intersect
model.load_state_dict(state_dict, strict=False) #load
else:
model = Model(opt.cfg, ch=3, nc=nc,
anchors=hyp.get('anchors')).to(device)
with torch_distributed_zero_first(rank):
check_dataset(data_dict) #check
train_path = data_dict['train']
test_path = data_dict['val']
# Optimizer
nbs = 64
accumulate = max(round(nbs / total_batch_size),
1) # accumulate loss before optimizing
hyp['weight_decay'] *= total_batch_size * accumulate / nbs
pg0, pg1, pg2 = [], [], [] # optimizer parameter groups
for k, v in model.named_modules():
if hasattr(v, 'bias') and isinstance(v.bias, nn.Parameter):
pg2.append(v.bias) # biases
if isinstance(v, nn.BatchNorm2d):
pg0.append(v.weight) # no decay
elif hasattr(v, 'weight') and isinstance(v.weight, nn.Parameter):
pg1.append(v.weight) # apply dacay
if opt.adam:
optimizer = optim.Adam(pg0,
lr=hyp['lr0'],
betas=(hyp['momentum'],
0.999)) # adjust betal to momentum
else:
optimizer = optim.SGD(pg0,
lr=hyp['lr0'],
momentum=hyp['momentum'],
nesterov=True)
optimizer.add_param_group({
'params': pg1,
'weight_decay': hyp['weight_decay']
}) # add pg1 with weight_decay
optimizer.add_param_group({'params': pg2}) # add pg2 (biases)
del pg0, pg1, pg2
if opt.linear_lr:
lf = lambda x: (1 - x / (epochs - 1)) * (1.0 - hyp['lrf']) + hyp[
'lrf'] # linear
else:
lf = one_cycle(1, hyp['lrf'], epochs)
scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf)
# EMA
ema = ModelEMA(model) if rank in [-1, 0] else None
# Resume
start_epoch, best_fitness = 0, 0.0
if pretrained:
# optimizer
if ckpt['optimizer'] is not None:
optimizer.load_state_dict(ckpt['optimizer'])
best_fitness = ckpt['best_fitness']
# EMA
if ema and ckpt.get('ema'):
ema.ema.load_state_dict(ckpt['ema'].float().state_dict())
ema.updates = ckpt['updates']
# Results
if ckpt.get('training_results') is not None:
results_file.write_text(
ckpt['training_results']) # write results.txt
# Epochs
start_epoch = ckpt['epoch'] + 1
if opt.resume:
assert start_epoch > 0, '%s training to %g epochs is finished, nothing to resume.' % (
weight, epochs)
if epochs < start_epoch:
epochs += ckpt['epoch']
del ckpt, state_dict
# Image sizes
gs = max(int(model.stride.max()), 32) # grid size (max stride)
nl = model.model[
-1].nl # number of detection layer (used for scaling hyp['obj])
imgsz, imgsz_test = [check_img_size(x, gs) for x in opt.img_size
] # verify imgsz are gs-multiples
# DP mode
if cuda and rank == -1 and torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
# SyncBatchNorm
if opt.sync_bn and cuda and rank != -1:
model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model).to(
device)
# Trainloader
if do_semi:
dataloader, dataset, unlabeldataloader = create_dataloader(
train_path,
imgsz,
batch_size,
gs,
opt,
hyp=hyp,
augment=True,
cache=opt.cache_images,
rect=opt.rect,
rank=rank,
world_size=opt.world_size,
workers=opt.workers,
image_weights=opt.image_weights,
quad=opt.quad,
prefix=colorstr('train: '),
do_semi=opt.do_semi)
else:
dataloader, dataset = create_dataloader(
train_path,
imgsz,
batch_size,
gs,
opt,
hyp=hyp,
augment=True,
cache=opt.cache_images,
rect=opt.rect,
rank=rank,
world_size=opt.world_size,
workers=opt.workers,
image_weights=opt.image_weights,
quad=opt.quad,
prefix=colorstr('train: '),
do_semi=opt.do_semi)
# Train teacher model
mlc = np.concatenate(dataset.labels, 0)[:, 0].max() # max label class
nb = len(dataloader) # number of batches
assert mlc < nc, 'Label class %g exceeds nc=%g in %s. Possible class labels are 0-%g' % (
mlc, nc, opt.data, nc - 1)
# process 0
if rank in [-1, 0]:
testloader = create_dataloader(
test_path,
imgsz_test,
batch_size * 2,
gs,
opt, # testloader
hyp=hyp,
cache=opt.cache_images and not opt.notest,
rect=True,
rank=-1,
world_size=opt.world_size,
workers=opt.workers,
pad=0.5,
prefix=colorstr('val: '),
do_semi=False)[0]
if not opt.resume:
labels = np.concatenate(dataset.labels, 0)
c = torch.tensor(labels[:, 0]) # classes
# Anchors
if not opt.noautoanchor:
check_anchors(dataset,
model=model,
thr=hyp['anchor_t'],
imgsz=imgsz)
model.half().float() # pre-reduce anchor precision
# DDP mode
if cuda and rank != 1:
model = DDP(model,
device_ids=[opt.local_rank],
output_device=opt.local_rank,
find_unused_parameters=any(
isinstance(layer, nn.MultiheadAttention)
for layer in model.modules()))
# Model parameters
hyp['box'] *= 3. / nl # scale to layers
hyp['cls'] *= nc / 80. * 3. / nl # scale to classes and layers
hyp['obj'] *= (imgsz / 640)**2 * 3. / nl # scale to image size and layers
hyp['label_smoothing'] = opt.label_smoothing
model.nc = nc # attach number of classes to model
model.hyp = hyp # attach hyperparameters to model
model.gr = 1.0 # iou loss ratio (obj_loss = 1.0 or iou)
model.class_weights = labels_to_class_weights(
dataset.labels, nc).to(device) * nc # attach class weights
model.names = names
# Train teacher model --> burn in
t0 = time.time()
nw = max(round(hyp['warmup_epochs'] * nb),
1000) # number of warmup iterations, max(3 epochs, 1k iterations)
# nw = min(nw, (epochs - start_epoch) / 2 * nb) # limit warmup to < 1/2 of training
maps = np.zeros(nc) # mAP per class
results = (0, 0, 0, 0, 0, 0, 0
) # P, R, [email protected], [email protected], val_loss(box, obj, cls)
scheduler.last_epoch = start_epoch - 1 # do not move
scaler = amp.GradScaler(enabled=cuda)
compute_loss = ComputeLoss(model) # init loss class
burnin_epochs = epochs / 2
# burn in
for epoch in range(start_epoch,
burnin_epochs): # epoch-------------------------
model.train()
nb = len(dataloader)
mloss = torch.zeros(4, device=device) # mean loss
if rank != -1:
dataloader.sampler.set_epoch(epoch)
pbar = enumerate(dataloader)
if rank in [-1, 0]:
pbar = tqdm(pbar, total=nb)
optimizer.zero_grad()
for i, (imgs, targets, paths, _) in pbar:
ni = i + nb * epoch # number integrated batches (since train start)
imgs = imgs.to(device, non_blocking=True).float(
) / 255.0 # uint8 to float32, 0-255 to 0.0-1.0
# Warm up
if ni <= [0, nw]:
xi = [0, nw]
accumulate = max(
1, np.interp(ni, xi, [1, nbs / total_batch_size].round()))
for j, x in enumerate(optimizer.param_groups):
# bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0
x['lr'] = np.interp(ni, xi, [
hyp['warmup_bias_lr'] if j == 2 else 0.0,
x['initial_lr'] * lf(epoch)
])
if 'momentum' in x:
x['momentum'] = np.interp(
ni, xi, [hyp['warmup_momentum'], hyp['momentum']])
# Forward
with amp.autocast(enabled=cuda):
pred = model(imgs) # forward
loss, loss_item = compute_loss(
pred, targets.to(device)) # loss scaled by batch_size
if rank != -1:
loss *= opt.world_size # gradient averaged between device in DDP mode
if opt.quad:
loss *= 4.
# Backward
scaler.scale(loss).backward()
# Optimize
if ni % accumulate == 0:
scaler.step(optimizer)
scaler.update()
optimizer.zero_grad()
if ema:
ema.update(model)
# print
if rank in [-1, 0]:
mloss = (mloss * i + loss_item) / (i + 1) # update mean losses
mem = '%.3gG' % (torch.cuda.memory_reserved() / 1E9
if torch.cuda.is_available() else 0) # (GB)
s = ('%10s' * 2 +
'%10.4g' * 6) % ('%g/%g' % (epoch, epochs - 1), mem,
*mloss, targets.shape[0], imgs.shape[-1])
pbar.set_description(s)
scheduler.step()
# DDP process 0 or single-GPU
if rank in [-1, 0]:
# mAP
ema.update_attr(model,
include=[
'yaml', 'nc', 'hyp', 'gr', 'names', 'stride',
'class_weights'
])
final_epoch = epoch + 1 == epochs
if not opt.notest or final_epoch: # Calculate mAP
results, maps, times = test.test(data_dict,
batch_size=batch_size * 2,
imgsz=imgsz_test,
model=ema.ema,
single_cls=opt.single_cls,
dataloader=testloader,
save_dir=save_dir,
verbose=nc < 50
and final_epoch,
plots=plots and final_epoch,
compute_loss=compute_loss)
fi = fitness(np.array(results).reshape(
1, -1)) # weighted combination of [P, R, mAP@50, [email protected]]
if fi > best_fitness:
best_fitness = fi
if (not opt.nosave) or (final_epoch and not opt.evolve): # if save
ckpt = {
'epoch':
epoch,
'best_fitness':
best_fitness,
'training_results':
results_file.read_text(),
'model':
deepcopy(model.module if is_parallel(model) else model).half(),
'ema':
deepcopy(ema.ema).half(),
'updates':
ema.updates,
'optimizer':
optimizer.state_dict()
}
if best_fitness == fi:
torch.save(ckpt, best)
del ckpt
# end epoch ----------------------------------------------------------------------------
# end warm up
# get persudo label
# STAC
# first apply weak augmentation on unlabeled dataset then use teacher net to predict the persudo labels
# Then apply strong augmentation on unlabeled dataset, use student net to get the logists and compute the unlabeled loss.
model.eval()
img = []
target = []
Path = []
imgsz = opt.img_size
for idx, batch in tqdm(enumerate(unlabeldataloader),
total=len(unlabeldataloader)):
imgs0, _, path, _ = batch # from uint8 to float16
with torch.no_grad():
pred = model(imgs0.to(device, non_blocking=True).float() /
255.0)[0]
gn = torch.tensor(imgs0.shape)[[3, 2, 3, 2]]
pred = non_max_suppression(pred,
opt.conf_thres,
opt.iou_thres,
classes=opt.classes,
agnostic=opt.agnostic_nms)
for index, pre in enumerate(pred):
predict_number = len(pre)
if predict_number == 0:
continue
Class = pre[:, 5].view(predict_number, 1).cpu()
XYWH = (xyxy2xywh(pre[:, :4])).cpu()
XYWH /= gn
pre = torch.cat((torch.zeros(predict_number, 1), Class, XYWH),
dim=1)
img.append(imgs0[index])
target.append(pre)
Path.append(path[index])
unlabeldataset = semiDataset(img, target, Path)
del img, targets, Path
model.train()
return os.path.exists(h_path)
def add_version_header(tc):
tc["headers"].append("version")
return tc
feature_test_macros = sorted(
[
add_version_header(x) for x in [
# C++14 macros
{
"name": "__cpp_lib_integer_sequence",
"values": {
"c++14": int(201304)
},
"headers": ["utility"],
},
{
"name": "__cpp_lib_exchange_function",
"values": {
"c++14": int(201304)
},
"headers": ["utility"],
},
{
"name": "__cpp_lib_tuples_by_type",
"values": {
"c++14": int(201304)
},
def isinf(self):
""" Returns True if the element is infinity."""
return int(_C.G2_ELEM_is_at_infinity(self.group.bpg, self.elem)) == 1
def process_query_result(self, html, searchRequest, maxResults=None):
self.debug("Started processing results")
entries = []
countRejected = self.getRejectedCountDict()
logger.debug("Using HTML parser %s" %
config.settings.searching.htmlParser)
soup = BeautifulSoup(html, config.settings.searching.htmlParser)
main_table = soup.find(id="results").find('table')
if "No results found" in soup.text:
return IndexerProcessingResult(
entries=[],
queries=[],
total=0,
total_known=True,
has_more=False,
rejected=self.getRejectedCountDict())
if not main_table or not main_table.find("tbody"):
self.error("Unable to find main table in NZBIndex page: %s..." %
html[:500])
self.debug(html[:500])
raise IndexerResultParsingException(
"Unable to find main table in NZBIndex page", self)
items = main_table.find("tbody").find_all('tr')
for row in items:
try:
entry = self.parseRow(row)
except IndexerResultParsingRowException:
continue
accepted, reason, ri = self.accept_result(entry, searchRequest,
self.supportedFilters)
if accepted:
entries.append(entry)
else:
countRejected[ri] += 1
self.debug("Rejected search result. Reason: %s" % reason)
try:
page_links = main_table.find("tfoot").find_all("tr")[1].find_all(
'a')
if len(page_links) == 0:
total = len(entries)
has_more = False
else:
pagecount = int(page_links[-2].text)
currentpage = int(
main_table.find("tfoot").find_all("tr")[1].find(
"b").text) #Don't count "next"
has_more = pagecount > currentpage
total = self.limit * pagecount #Good enough
except Exception:
self.exception("Error while trying to find page count")
total = len(entries)
has_more = False
self.debug("Finished processing results")
return IndexerProcessingResult(entries=entries,
queries=[],
total=total,
total_known=True,
has_more=has_more,
rejected=countRejected)
def _generate_standard_design(self, infolist, functional_runs=None,
realignment_parameters=None, outliers=None):
""" Generates a standard design matrix paradigm given information about
each run
"""
sessinfo = []
output_units = 'secs'
if 'output_units' in self.inputs.traits():
output_units = self.inputs.output_units
for i, info in enumerate(infolist):
sessinfo.insert(i, dict(cond=[]))
if isdefined(self.inputs.high_pass_filter_cutoff):
sessinfo[i]['hpf'] = \
np.float(self.inputs.high_pass_filter_cutoff)
if hasattr(info, 'conditions') and info.conditions is not None:
for cid, cond in enumerate(info.conditions):
sessinfo[i]['cond'].insert(cid, dict())
sessinfo[i]['cond'][cid]['name'] = info.conditions[cid]
scaled_onset = scale_timings(info.onsets[cid],
self.inputs.input_units,
output_units,
self.inputs.time_repetition)
sessinfo[i]['cond'][cid]['onset'] = scaled_onset
scaled_duration = scale_timings(info.durations[cid],
self.inputs.input_units,
output_units,
self.inputs.time_repetition)
sessinfo[i]['cond'][cid]['duration'] = scaled_duration
if hasattr(info, 'amplitudes') and info.amplitudes:
sessinfo[i]['cond'][cid]['amplitudes'] = \
info.amplitudes[cid]
if hasattr(info, 'tmod') and info.tmod and \
len(info.tmod) > cid:
sessinfo[i]['cond'][cid]['tmod'] = info.tmod[cid]
if hasattr(info, 'pmod') and info.pmod and \
len(info.pmod) > cid:
if info.pmod[cid]:
sessinfo[i]['cond'][cid]['pmod'] = []
for j, name in enumerate(info.pmod[cid].name):
sessinfo[i]['cond'][cid]['pmod'].insert(j, {})
sessinfo[i]['cond'][cid]['pmod'][j]['name'] = \
name
sessinfo[i]['cond'][cid]['pmod'][j]['poly'] = \
info.pmod[cid].poly[j]
sessinfo[i]['cond'][cid]['pmod'][j]['param'] = \
info.pmod[cid].param[j]
sessinfo[i]['regress'] = []
if hasattr(info, 'regressors') and info.regressors is not None:
for j, r in enumerate(info.regressors):
sessinfo[i]['regress'].insert(j, dict(name='', val=[]))
if hasattr(info, 'regressor_names') and \
info.regressor_names is not None:
sessinfo[i]['regress'][j]['name'] = \
info.regressor_names[j]
else:
sessinfo[i]['regress'][j]['name'] = 'UR%d' % (j + 1)
sessinfo[i]['regress'][j]['val'] = info.regressors[j]
sessinfo[i]['scans'] = functional_runs[i]
if realignment_parameters is not None:
for i, rp in enumerate(realignment_parameters):
mc = realignment_parameters[i]
for col in range(mc.shape[1]):
colidx = len(sessinfo[i]['regress'])
sessinfo[i]['regress'].insert(colidx, dict(name='', val=[]))
sessinfo[i]['regress'][colidx]['name'] = 'Realign%d' % (col + 1)
sessinfo[i]['regress'][colidx]['val'] = mc[:, col].tolist()
if outliers is not None:
for i, out in enumerate(outliers):
numscans = 0
for f in filename_to_list(sessinfo[i]['scans']):
shape = load(f, mmap=NUMPY_MMAP).shape
if len(shape) == 3 or shape[3] == 1:
iflogger.warning('You are using 3D instead of 4D '
'files. Are you sure this was '
'intended?')
numscans += 1
else:
numscans += shape[3]
for j, scanno in enumerate(out):
colidx = len(sessinfo[i]['regress'])
sessinfo[i]['regress'].insert(colidx, dict(name='', val=[]))
sessinfo[i]['regress'][colidx]['name'] = 'Outlier%d' % (j + 1)
sessinfo[i]['regress'][colidx]['val'] = \
np.zeros((1, numscans))[0].tolist()
sessinfo[i]['regress'][colidx]['val'][int(scanno)] = 1
return sessinfo
'''
Created on Jul 19, 2018
@author: Yan
'''
from builtins import int
print('Enter birth Year')
birthYear = int(input('It must be an integer\n'))
def ageCalculator(birthYear):
age = 2018 - birthYear
print('your age is: ', age)
return age
def test_correct_age_is_calculated(calculatedAge):
assert currentAge == calculatedAge
print('\nAssertion made for: ', currentAge, ' years old!')
return True
currentAge = ageCalculator(birthYear)
testAge = test_correct_age_is_calculated(currentAge)
def f_plane_wind_test(physics, aro_exec, nx, ny, dx, dy, dt, nTimeSteps):
layers = 1
grid = aro.Grid(nx, ny, layers, dx, dy)
rho0 = 1035.
def wind_x(X, Y, *arg):
wind_x = np.zeros(Y.shape, dtype=np.float64)
wind_x[int(grid.nx / 2), int(grid.ny / 2)] = 1e-5
if not arg:
plt.figure()
plt.pcolormesh(X / 1e3, Y / 1e3, wind_x)
plt.colorbar()
plt.savefig('wind_x.png')
plt.close()
return wind_x
def wind_y(X, Y, *arg):
wind_y = np.zeros(X.shape, dtype=np.float64)
wind_y[int(grid.nx / 2), int(grid.ny / 2)] = 1e-5
if not arg:
plt.figure()
plt.pcolormesh(X / 1e3, Y / 1e3, wind_y)
plt.colorbar()
plt.savefig('wind_y.png')
plt.close()
return wind_y
def dbl_periodic_wetmask(X, Y):
return np.ones(X.shape, dtype=np.float64)
with opt.working_directory(
p.join(self_path,
"physics_tests/f_plane_{0}_wind".format(physics))):
sub.check_call(["rm", "-rf", "output/"])
drv.simulate(initHfile=[400.],
zonalWindFile=[wind_x],
meridionalWindFile=[wind_y],
valgrind=False,
nx=nx,
ny=ny,
exe=aro_exec,
dx=dx,
dy=dy,
wetMaskFile=[dbl_periodic_wetmask],
dt=dt,
dumpFreq=int(dt * nTimeSteps / 50),
nTimeSteps=nTimeSteps)
hfiles = sorted(glob.glob("output/snap.h.*"))
ufiles = sorted(glob.glob("output/snap.u.*"))
vfiles = sorted(glob.glob("output/snap.v.*"))
# expect the momentum to grow according to u*h*rho0 = delta_t * wind
# F = m * a
# m * v = h * rho0 * xlen * ylen * v
# = m * a * dt
# = F * dt
# = wind * dx * dy * dt
momentum = np.zeros(len(hfiles), dtype=np.float64)
model_iteration = np.zeros(len(hfiles), dtype=np.float64)
momentum_expected = np.zeros(len(hfiles), dtype=np.float64)
volume = np.zeros(len(hfiles))
for counter, ufile in enumerate(ufiles):
h = aro.interpret_raw_file(hfiles[counter], nx, ny, layers)
u = aro.interpret_raw_file(ufile, nx, ny, layers)
v = aro.interpret_raw_file(vfiles[counter], nx, ny, layers)
model_iteration[counter] = float(ufile[-10:])
# plt.figure()
# plt.pcolormesh(grid.xp1,grid.y,u[:,:,0].transpose())
# plt.colorbar()
# plt.savefig('u.{0}.png'.format(ufile[-10:]),dpi=150)
# plt.close()
# plt.figure()
# plt.pcolormesh(grid.x,grid.y,h[:,:,0].transpose())
# plt.colorbar()
# plt.savefig('h.{0}.png'.format(ufile[-10:]),dpi=150)
# plt.close()
momentum[counter] = dx * dy * rho0 * (
np.sum(np.absolute(h * (u[:, :, 1:] + u[:, :, :-1]) / 2.)) +
np.sum(np.absolute(h * (v[:, 1:, :] + v[:, :-1, :]) / 2.)))
momentum_expected[counter] = 2. * dx * dy * 1e-5 * (
model_iteration[counter] + 2) * dt
volume[counter] = np.sum(dx * dy * h)
# plt.figure()
# plt.pcolormesh(grid.xp1, grid.y, np.transpose(u[:,:,0]))
# plt.colorbar()
# plt.savefig('output/u.{0}.png'.format(model_iteration[counter]),dpi=100)
# plt.close()
opt.assert_volume_conservation(nx, ny, layers, 1e-9)
plt.figure()
plt.plot(model_iteration * dt / (30 * 86400),
momentum_expected,
'-',
alpha=1,
label='Expected momentum')
plt.plot(model_iteration * dt / (30 * 86400),
momentum,
'-',
alpha=1,
label='Simulated momentum')
plt.legend()
plt.xlabel('Time (months)')
plt.ylabel('Momentum')
plt.savefig('f_plane_momentum_test.png', dpi=150)
plt.close()
plt.figure()
plt.plot(model_iteration, momentum / momentum_expected)
plt.xlabel('timestep')
plt.ylabel('simulated/expected')
plt.title('final ratio = {0}'.format(
str(momentum[-1] / momentum_expected[-1])))
plt.savefig('ratio.png')
plt.close()
plt.figure()
plt.plot(model_iteration,
100. * (momentum - momentum_expected) / momentum_expected)
plt.xlabel('timestep')
plt.ylabel('percent error')
plt.ylim(-4, 4)
plt.savefig('percent_error.png')
plt.close()
plt.figure()
plt.plot(model_iteration, momentum - momentum_expected)
plt.xlabel('timestep')
plt.ylabel('simulated - expected')
plt.savefig('difference.png')
plt.close()
plt.figure()
plt.plot(model_iteration, volume)
plt.ylabel('Volume')
plt.xlabel('timestep')
plt.ylim(np.min(volume), np.max(volume))
plt.savefig('volume.png')
plt.close()
percent_error = 100. * (momentum -
momentum_expected) / momentum_expected
return percent_error[-1]
def set_int_custom_property(self, key: Text, value: int):
"""Set a custom property of int type."""
self._artifact.custom_properties[key].int_value = builtins.int(value)
def configFeAsic(self):
print("CONFIG ASICs")
# #global config varibles
feasicLeakageVal = int( self.feasicLeakage ) #0 = 500pA, 1 = 100pA
feasicLeakagex10Val = int( self.feasicLeakagex10 ) #0 = x1, 1 = x10
acdcVal = int( self.feasicAcdc ) #DC = 0, AC = 1
#channel specific variables
testVal = int( self.feasicEnableTestInput )
baseVal = int( self.feasicBaseline ) #0 = 900mV, 1 = 200mV
gainVal = int( self.feasicGain )
shapeVal = int( self.feasicShape )
bufVal = int( self.feasicBuf ) #0 = OFF, 1 = ON
if (testVal < 0 ) or (testVal > 1):
return
if (baseVal < 0 ) or (baseVal > 1):
return
if (gainVal < 0 ) or (gainVal > 3):
return
if (shapeVal < 0 ) or (shapeVal > 3):
return
if (acdcVal < 0 ) or (acdcVal > 1):
return
if (bufVal < 0 ) or (bufVal > 1):
return
if (feasicLeakageVal < 0 ) or (feasicLeakageVal > 1 ):
return
if (feasicLeakagex10Val < 0) or (feasicLeakagex10Val > 1):
return
#gain
gainArray = [0,2,1,3] #00=4.7, 10=7.8, 01=14, 11=25
gainValCorrect = gainArray[gainVal]
#shape
shapeArray = [2,0,3,1] #00=1.0, 10=0.5, 01=3.0, 11=2.0
shapeValCorrect = shapeArray[shapeVal]
#COTS Register Settings
sts = testVal
snc = baseVal
sg = gainValCorrect
st = shapeValCorrect
smn = 0 #Output monitor enabled: not currently an option in femb_python so keep at 0 for now
sdf = bufVal
chn_reg = ((sts&0x01)<<7) + ((snc&0x01)<<6) + ((sg&0x03)<<4) + ((st&0x03)<<2) + ((smn&0x01)<<1) + ((sdf&0x01)<<0)
#COTS Global Register Settings
slk0 = feasicLeakageVal
stb1 = 0 #Monitors not currently used in femb_python
stb = 0 #Monitors not currently used in femb_python
s16 = 0 #High filter in channel 16 disabled
slk1 = feasicLeakagex10Val
sdc = acdcVal
swdac = 0 #For pulser, set elsewhere
dac = 0 #For pulser, set elsewhere
global_reg = ((slk0&0x01)<<0) + ((stb1&0x01)<<1) + ((stb&0x01)<<2)+ ((s16&0x01)<<3) + ((slk1&0x01)<<4) + ((sdc&0x01)<<5) +((00&0x03)<<6)
dac_reg = (((dac&0x01)//0x01)<<7)+(((dac&0x02)//0x02)<<6)+\
(((dac&0x04)//0x04)<<5)+(((dac&0x08)//0x08)<<4)+\
(((dac&0x10)//0x10)<<3)+(((dac&0x20)//0x20)<<2)+\
(((swdac&0x03))<<0)
for chip in range(self.NASICS):
for chn in range(self.NASICCH):
if self.useLArIATmap:
key = "wib{:d}_femb{:d}_chip{:d}_chan{:02d}".format(self.wibNum,self.fembNum,chip+1,chn) #Note map has chips 1-8, not 0-7
if self.WireDict[key][0] == "X":
snc = 0 #set baseline for collection
elif self.WireDict[key][0] == "U":
snc = 1 #set baseline for induction
chn_reg = ((sts&0x01)<<7) + ((snc&0x01)<<6) + ((sg&0x03)<<4) + ((st&0x03)<<2) + ((smn&0x01)<<1) + ((sdf&0x01)<<0)
chn_reg_bool = []
for j in range(8):
chn_reg_bool.append ( bool( (chn_reg>>j)%2 ))
start_pos = (8*16+16)*chip + (16-chn)*8
self.fe_regs[start_pos-8 : start_pos] = chn_reg_bool
global_reg_bool = []
for j in range(8):
global_reg_bool.append ( bool( (global_reg>>j)%2 ) )
for j in range(8):
global_reg_bool.append ( bool( (dac_reg>>j)%2 ) )
start_pos = (8*16+16)*chip + 16*8
self.fe_regs[start_pos : start_pos+16] = global_reg_bool
#Convert bits to 36 32-bit register words
for chip in [0,2,4,6]:
chip_bits_len = 8*(16+2)
chip_fe_regs0 = self.fe_regs[ chip*chip_bits_len: (chip+1)* chip_bits_len]
chip_fe_regs1 = self.fe_regs[ (chip+1)*chip_bits_len: (chip+2)* chip_bits_len]
chip_regs = []
for onebit in chip_fe_regs0:
chip_regs.append(onebit)
for onebit in chip_fe_regs1:
chip_regs.append(onebit)
len32 = len(chip_regs)//32
if (len32 != 9):
print("ERROR FE register mapping")
else:
for i in range(len32):
if ( i*32 <= len(chip_regs) ):
bits32 = chip_regs[i*32: (i+1)*32]
self.fe_REGS[int(chip/2*len32 + i) ] = (sum(v<<j for j, v in enumerate(bits32)))
#turn off HS data before register writes
self.femb.write_reg_bits(9 , 0, 0x1, 0 )
print("HS link turned off")
time.sleep(2)
#run the SPI programming
self.doAsicConfig()
#turn HS link back on
print("HS link turned back on")
time.sleep(2)
self.femb.write_reg_bits(9 , 0, 0x1, 1 )
def __init__(self):
#declare basic system parameters
self.NFEMBS = 4
self.NASICS = 8
self.NASICCH = 16
#declare board specific registers
self.FEMB_VER = "WIB_SBND"
self.REG_RESET = 0
self.REG_ASIC_RESET = 1
self.REG_ASIC_SPIPROG = 2
self.REG_SOFT_ADC_RESET = 1
self.REG_LATCHLOC_3_TO_0 = 4
self.REG_LATCHLOC_7_TO_4 = 14
self.REG_FPGA_TP_EN = 16
self.REG_ASIC_TP_EN = 16
self.REG_DAC_SELECT = 16
self.REG_TP = 5
self.CLK_SELECT = 6
self.CLK_SELECT2 = 15
self.REG_SEL_ASIC = 7
self.REG_SEL_ASIC_LSB = 8
self.REG_WIB_MODE = 8
self.REG_ADC_DISABLE = 8
self.REG_HS_DATA = 9
self.REG_HS = 17
self.INT_TP_EN = 18
self.EXT_TP_EN = 18
self.REG_SPI_BASE = 0x200
self.REG_SPI_RDBACK_BASE = 0x250
self.REG_TEST_PAT = 3
self.REG_TEST_PAT_DATA = 0x01230000
#COTS shifts
self.fe1_sft = 0x00000000
self.fe2_sft = 0x00000000
self.fe3_sft = 0x00000000
self.fe4_sft = 0x00000000
self.fe5_sft = 0x00000000
self.fe6_sft = 0x00000000
self.fe7_sft = 0x00000000
self.fe8_sft = 0x00000000
#COTS phases
self.fe1_pha = 0x00000000
self.fe2_pha = 0x00000000
self.fe3_pha = 0x00000000
self.fe4_pha = 0x00000000
self.fe5_pha = 0x00000000
self.fe6_pha = 0x00000000
self.fe7_pha = 0x00000000
self.fe8_pha = 0x00000000
#internal variables
self.fembNum = 0
self.wibNum = 0
self.useExtAdcClock = True
self.isRoomTemp = False
self.doReSync = True
self.spiStatus = 0x0
self.syncStatus = 0x0
self.CLKSELECT_val_RT = 0xFF
self.CLKSELECT2_val_RT = 0xFF
self.CLKSELECT_val_CT = 0xEF
self.CLKSELECT2_val_CT = 0xEF
self.REG_LATCHLOC_3_TO_0_val = 0x04040404
self.REG_LATCHLOC_7_TO_4_val = 0x04040404
self.fe_regs = [0x00000000]*(16+2)*8*8
self.fe_REGS = [0x00000000]*(8+1)*4
self.useLArIATmap = True
#initialize FEMB UDP object
self.femb = FEMB_UDP()
self.femb.UDP_PORT_WREG = 32000 #WIB PORTS
self.femb.UDP_PORT_RREG = 32001
self.femb.UDP_PORT_RREGRESP = 32002
self.femb.doReadBack = False #WIB register interface is unreliable
#ASIC config variables
self.feasicLeakage = 0 #0 = 500pA, 1 = 100pA
self.feasicLeakagex10 = 0 #0 = pA, 1 = pA*10
self.feasicAcdc = 0 #AC = 0, DC = 1
self.feasicBaseline = 1 #0 = 200mV, 1 = 900mV
self.feasicEnableTestInput = 0 #0 = disabled, 1 = enabled
self.feasicGain = 2 #4.7,7.8,14,25
self.feasicShape = 1 #0.5,1,2,3
self.feasicBuf = 0 #0 = OFF, 1 = ON
#Read in LArIAT mapping if desired
if self.useLArIATmap:
self.cppfr = CPP_FILE_RUNNER()
with open(self.cppfr.filename('configuration/configs/LArIAT_pin_mapping.map'), "rb") as fp:
self.lariatMap = pickle.load(fp)
#APA Mapping
va = self.lariatMap
va_femb = []
for vb in va:
if int(vb[9]) in (0,1,2,3,4) :
va_femb.append(vb)
apa_femb_loc = []
for chn in range(128):
for vb in va_femb:
if int(vb[8]) == chn:
if (vb[1].find("Co")) >= 0 :#collection wire
chninfo = [ "X" + vb[0], vb[8], int(vb[6]), int(vb[7]), int(vb[9]), int(vb[10])]
elif (vb[1].find("In")) >= 0 : #induction wire
chninfo = [ "U" + vb[0], vb[8], int(vb[6]), int(vb[7]), int(vb[9]), int(vb[10])]
apa_femb_loc.append(chninfo)
for chn in range(128):
fl_w = True
fl_i = 0
for tmp in apa_femb_loc:
if int(tmp[1]) == chn:
fl_w = False
break
if (fl_w):
chninfo = [ "V" + format(fl_i, "03d"), format(chn, "03d"), chn//16 , format(chn%15, "02d"), apa_femb_loc[0][4], apa_femb_loc[0][5]]
apa_femb_loc.append(chninfo)
fl_i = fl_i + 1
self.All_sort = []
self.X_sort = []
self.V_sort = []
self.U_sort = []
for i in range(128):
for chn in apa_femb_loc:
if int(chn[1][0:3]) == i :
self.All_sort.append(chn)
for chn in apa_femb_loc:
if chn[0][0] == "X" and int(chn[0][1:3]) == i :
self.X_sort.append(chn)
for chn in apa_femb_loc:
if chn[0][0] == "V" and int(chn[0][1:3]) == i :
self.V_sort.append(chn)
for chn in apa_femb_loc:
if chn[0][0] == "U" and int(chn[0][1:3]) == i :
self.U_sort.append(chn)
self.WireDict = {}
for line in self.All_sort:
key = "wib{:d}_femb{:d}_chip{:d}_chan{:02d}".format(line[5],line[4],line[2],line[3])
self.WireDict[key] = line[0]
def _ReturnSessionIdValue(self, InputString):
#Session Id: 0xF08A4
return int(InputString.partition(":")[2].strip(), base=0)
def __init__(self, dtype_or_func=None, default=None, missing_values=None,
locked=False):
# Convert unicode (for Py3)
if isinstance(missing_values, unicode):
missing_values = asbytes(missing_values)
elif isinstance(missing_values, (list, tuple)):
missing_values = asbytes_nested(missing_values)
# Defines a lock for upgrade
self._locked = bool(locked)
# No input dtype: minimal initialization
if dtype_or_func is None:
self.func = str2bool
self._status = 0
self.default = default or False
dtype = np.dtype('bool')
else:
# Is the input a np.dtype ?
try:
self.func = None
dtype = np.dtype(dtype_or_func)
except TypeError:
# dtype_or_func must be a function, then
if not hasattr(dtype_or_func, '__call__'):
errmsg = ("The input argument `dtype` is neither a"
" function nor a dtype (got '%s' instead)")
raise TypeError(errmsg % type(dtype_or_func))
# Set the function
self.func = dtype_or_func
# If we don't have a default, try to guess it or set it to
# None
if default is None:
try:
default = self.func(asbytes('0'))
except ValueError:
default = None
dtype = self._getdtype(default)
# Set the status according to the dtype
_status = -1
for (i, (deftype, func, default_def)) in enumerate(self._mapper):
if np.issubdtype(dtype.type, deftype):
_status = i
if default is None:
self.default = default_def
else:
self.default = default
break
if _status == -1:
# We never found a match in the _mapper...
_status = 0
self.default = default
self._status = _status
# If the input was a dtype, set the function to the last we saw
if self.func is None:
self.func = func
# If the status is 1 (int), change the function to
# something more robust.
if self.func == self._mapper[1][1]:
if issubclass(dtype.type, np.uint64):
self.func = np.uint64
elif issubclass(dtype.type, np.int64):
self.func = np.int64
else:
self.func = lambda x: int(float(x))
# Store the list of strings corresponding to missing values.
if missing_values is None:
self.missing_values = set([asbytes('')])
else:
if isinstance(missing_values, bytes):
missing_values = missing_values.split(asbytes(","))
self.missing_values = set(list(missing_values) + [asbytes('')])
#
self._callingfunction = self._strict_call
self.type = self._dtypeortype(dtype)
self._checked = False
self._initial_default = default
# c: 0xC03DDDA6174963AD10224BADDBCF7ED9EA5E3DAE91941CB428D2EC060B4F290A
# u1: 0x113BE17918E856E4D6EC2EE04F5E9B3CB599B82AC879C8E32A0140C290D32659
# u2: 0x2976F786AE6333E125C0DFFD6C16D37E8CED5ABEDB491BCCA21C75B307D0B318
# u1G: 0x51e4e6ed6f4b1db33b0d21b8bd30fb732f1d999c4e27bb1800eba20813ad3e86
# 0x93101a9fa0d5c7c680400b03d3becb9130dd8f9f4d9b034360a74829dc1201ab
# u2Q: 0xeaca8440897333e259d0f99165611b085d6e10a9bfd371c451bc0aea1aeb99c3
# 0x57c5c95ea9f491c0fd9029a4089a2e6df47313f915f3e39e9f12e03ab16521c2
# + : 0x0623b4159c7112125be51716d1e706d68e52f5b321da68d8b86b3c7c7019a9da
# : 0x1029094ccc466a534df3dbb7f588b283c9bef213633750aeff021c4c131b7ce5
# SIG: 3045
# 0220
# 0623b4159c7112125be51716d1e706d68e52f5b321da68d8b86b3c7c7019a9da
# 0221
# 008dffe3c592a0c7e5168dcb3d4121a60ee727082be4fbf79eae564929156305fc
msg = int(
0xba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad)
sig = int(
0x304502200623b4159c7112125be51716d1e706d68e52f5b321da68d8b86b3c7c7019a9da0221008dffe3c592a0c7e5168dcb3d4121a60ee727082be4fbf79eae564929156305fc
)
msg = msg.to_bytes(32, 'big')
sig = sig.to_bytes(0x47, 'big')
signer = ECDSA()
assert (signer.verify(msg, sig, pu_key))
# k: 0xe5a8d1d529971c10ca2af378444fb544a211707892c8898f91dcb171584e3db9
# kG: 0x4f123ed9de853836447782f0a436508d34e6609083cf97c9b9cd69673d8f04a5
# 0x50b57473f987f2d7c4715827dbd7b23c3088645d5f898aa66e4ef2778591d643
# kinv: 0x0F2DD0361F61F683957CF708FB54DBC0B6B97F9EDF28604983E6F492117C154C
# kinv.d 0xFF68C89B97C63273EB5F787FBC0C33DA02BA4C883AB09E3D381197E9E3964E8C
# kinv.d.x 0x5A20058782FB81C8F98C30D9D441B7196C22939B144918CF519FB155180664B1
return os.path.exists(h_path)
def add_version_header(tc):
tc["headers"].append("version")
return tc
feature_test_macros = sorted(
[
add_version_header(x) for x in [
# C++14 macros
{
"name": "__cpp_lib_integer_sequence",
"values": {
"c++14": int(201304)
},
"headers": ["utility"],
},
{
"name": "__cpp_lib_exchange_function",
"values": {
"c++14": int(201304)
},
"headers": ["utility"],
},
{
"name": "__cpp_lib_tuples_by_type",
"values": {
"c++14": int(201304)
},
def __init__(self,
server_address,
RequestHandlerClass,
bind_and_activate=True):
self.remove_file = None
self.mode = None
self.listen_fd = get_listen_fd()
if self.listen_fd:
server_address = self.listen_fd
self.address_family = socket.AF_UNIX
self.socket = socket.fromfd(self.listen_fd, socket.AF_UNIX,
socket.SOCK_STREAM)
elif server_address.startswith("unix:"):
self.address_family = socket.AF_UNIX
address = server_address[5:]
m = address.rfind(';mode=')
if m != -1:
self.mode = address[m + 6:]
address = address[:m]
if address[0] == '@':
address = address.replace('@', '\0', 1)
self.mode = None
else:
self.remove_file = address
server_address = address
self.socket = socket.socket(self.address_family, self.socket_type)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
elif server_address.startswith("tcp:"):
address = server_address[4:]
p = address.rfind(':')
if p != -1:
port = int(address[p + 1:])
address = address[:p]
else:
raise ConnectionError("Invalid address 'tcp:%s'" % address)
address = address.replace('[', '')
address = address.replace(']', '')
try:
res = socket.getaddrinfo(address,
port,
proto=socket.IPPROTO_TCP,
flags=socket.AI_NUMERICHOST)
except TypeError:
res = socket.getaddrinfo(address, port, self.address_family,
self.socket_type, socket.IPPROTO_TCP,
socket.AI_NUMERICHOST)
af, socktype, proto, canonname, sa = res[0]
self.address_family = af
self.socket_type = socktype
self.socket = socket.socket(self.address_family, self.socket_type)
server_address = sa[0:2]
else:
raise ConnectionError("Invalid address '%s'" % server_address)
BaseServer.__init__(self, server_address, RequestHandlerClass)
if bind_and_activate:
try:
self.server_bind()
self.server_activate()
except:
self.server_close()
raise
parser.add_argument("trail_size",
help="Display a trail of diameter x while rendering")
parser.add_argument("--performance_test",
help="don't render anything, just calculate",
action="store_true")
args = parser.parse_args()
system = None
if args.simulation_type == 'BarnesHut':
system = RenderableBarnesHutSystem()
elif args.simulation_type == 'BruteForce':
system = RenderableBruteForceSystem()
elif args.simulation_type == 'ApoapsisExample':
system = RenderableApoapsisExampleSystem()
elif args.simulation_type == 'HohmannExampleA':
system = RenderableHohmannTransferExampleSystemA()
elif args.simulation_type == 'HohmannExampleB':
system = RenderableHohmannTransferExampleSystemB()
elif args.simulation_type == 'HohmannExampleC':
system = RenderableHohmannTransferExampleSystemC()
else:
exit(1)
system.start_the_bodies(int(args.bodies))
renderer = SystemRenderer(system,
frames=int(args.frames),
trail_size=int(args.trail_size),
performance_test=args.performance_test)
renderer.run()
def intOrFloat(string):
'''Not sure if your string is formatted as an int or a float? Use intOrFloat instead!'''
try:
return int(string)
except ValueError:
return float(string)
def depinContent(u, PageSize,floadName):
urls = u
pagesize = int(PageSize)
for i in range(pagesize):
foladPath = floadName+"/"+str(i+1)
createfolad(foladPath)
Content = sendUrl(getMainUrl(urls,i+1,False)).content.decode('GBK')
#UI
if(floadName == 'UML'):
child = "UML"
if(floadName == '建模'):
child = "UI"
if(floadName == '需求'):
child = "requirement"
if(floadName == '设计'):
child = "swdesign"
if(floadName == '编码、构建与集成'):
child = "bmgjjc"
if(floadName == '测试'):
child = "test"
if(floadName == '界面'):
child = "UI"
if(floadName == '产品管理'):
child = "productmana"
if(floadName == '项目管理'):
child = "xmgl"
if(floadName == '研发管理'):
child = "productmana"
if(floadName == '配置管理'):
child = "pzgl"
if(floadName == '质量管理'):
child = "zlgl"
if(floadName == '过程改进'):
child = "process"
if(floadName == '大数据'):
child = "bigdata"
if(floadName == '数据库'):
child = "datebase"
if(floadName == '数据仓库'):
child = "datecangku"
if(floadName == '数据挖掘'):
child = "datecangku"
if(floadName == '企业架构'):
child = "qiyejiagou"
if(floadName == 'IT规划与治理'):
child = "ITguihua"
if(floadName == '运营管理'):
child = "yunyinggl"
if(floadName == 'IT运维'):
child = "itil"
if(floadName == 'DevOps'):
child = "devops"
if(floadName == '安全'):
child = "safe"
if(floadName == 'JAVA'):
child = "java"
if(floadName == '.net'):
child = "net"
if(floadName == 'c,c++'):
child = "C"
if(floadName == 'web开发'):
child = "web"
if(floadName == '移动端开发'):
child = "phone"
if(floadName == '嵌入式开发'):
child = "qrskf"
if(floadName == 'IT人员培养'):
child = "itpeiyang"
if(floadName == '云计算'):
child = "yjs"
if(floadName == '网络技术'):
child = "wljs"
if(floadName == '办公'):
child = "office"
if(floadName == '人工智能'):
child = "ai"
if(floadName == 'python'):
child = "python"
if(floadName == '微服务'):
child = "wfw"
if(floadName == '学生'):
child = ""
return
getContent(Content,foladPath, floadName,child)
def parse_target(target_name):
try:
host, ip = target_name.split(":")
return host, int(ip)
except ValueError:
raise ValueError("Target format is HOST:PORT")
from builtins import int
cadena = input("Dame una cadena: ")
i = int(input("Dame un numero: "))
j = int(input("Dame otro numero: "))
subcadena = ''
for k in range(i, j):
subcadena += cadena[k]
print("La subcadena entre %d y %d es %s" % (i, j, subcadena))
def isone(self):
""" Return zero if the element is one. """
return int(_C.GT_ELEM_is_unity(self.group.bpg, self.elem)) == 1
def eq(self, other):
""" Returns True if elements are equal. """
resp = _C.GT_ELEM_cmp(self.elem, other.elem)
return (int(resp) == 0)
def _gen_regress(self, i_onsets, i_durations, i_amplitudes, nscans):
"""Generates a regressor for a sparse/clustered-sparse acquisition
"""
bplot = False
if isdefined(self.inputs.save_plot) and self.inputs.save_plot:
bplot = True
import matplotlib
matplotlib.use(config.get('execution', 'matplotlib_backend'))
import matplotlib.pyplot as plt
TR = np.round(self.inputs.time_repetition * 1000) # in ms
if self.inputs.time_acquisition:
TA = np.round(self.inputs.time_acquisition * 1000) # in ms
else:
TA = TR # in ms
nvol = self.inputs.volumes_in_cluster
SCANONSET = np.round(self.inputs.scan_onset * 1000)
total_time = TR * (nscans - nvol) / nvol + TA * nvol + SCANONSET
SILENCE = TR - TA * nvol
dt = TA / 10.0
durations = np.round(np.array(i_durations) * 1000)
if len(durations) == 1:
durations = durations * np.ones((len(i_onsets)))
onsets = np.round(np.array(i_onsets) * 1000)
dttemp = gcd(TA, gcd(SILENCE, TR))
if dt < dttemp:
if dttemp % dt != 0:
dt = float(gcd(dttemp, dt))
if dt < 1:
raise Exception('Time multiple less than 1 ms')
iflogger.info('Setting dt = %d ms\n', dt)
npts = int(np.ceil(total_time / dt))
times = np.arange(0, total_time, dt) * 1e-3
timeline = np.zeros((npts))
timeline2 = np.zeros((npts))
if isdefined(self.inputs.model_hrf) and self.inputs.model_hrf:
hrf = spm_hrf(dt * 1e-3)
reg_scale = 1.0
if self.inputs.scale_regressors:
boxcar = np.zeros(int(50.0 * 1e3 / dt))
if self.inputs.stimuli_as_impulses:
boxcar[int(1.0 * 1e3 / dt)] = 1.0
reg_scale = float(TA / dt)
else:
boxcar[int(1.0 * 1e3 / dt):int(2.0 * 1e3 / dt)] = 1.0
if isdefined(self.inputs.model_hrf) and self.inputs.model_hrf:
response = np.convolve(boxcar, hrf)
reg_scale = 1.0 / response.max()
iflogger.info('response sum: %.4f max: %.4f', response.sum(),
response.max())
iflogger.info('reg_scale: %.4f', reg_scale)
for i, t in enumerate(onsets):
idx = int(np.round(t / dt))
if i_amplitudes:
if len(i_amplitudes) > 1:
timeline2[idx] = i_amplitudes[i]
else:
timeline2[idx] = i_amplitudes[0]
else:
timeline2[idx] = 1
if bplot:
plt.subplot(4, 1, 1)
plt.plot(times, timeline2)
if not self.inputs.stimuli_as_impulses:
if durations[i] == 0:
durations[i] = TA * nvol
stimdur = np.ones((int(durations[i] / dt)))
timeline2 = np.convolve(timeline2, stimdur)[0:len(timeline2)]
timeline += timeline2
timeline2[:] = 0
if bplot:
plt.subplot(4, 1, 2)
plt.plot(times, timeline)
if isdefined(self.inputs.model_hrf) and self.inputs.model_hrf:
timeline = np.convolve(timeline, hrf)[0:len(timeline)]
if isdefined(self.inputs.use_temporal_deriv) and \
self.inputs.use_temporal_deriv:
# create temporal deriv
timederiv = np.concatenate(([0], np.diff(timeline)))
if bplot:
plt.subplot(4, 1, 3)
plt.plot(times, timeline)
if isdefined(self.inputs.use_temporal_deriv) and \
self.inputs.use_temporal_deriv:
plt.plot(times, timederiv)
# sample timeline
timeline2 = np.zeros((npts))
reg = []
regderiv = []
for i, trial in enumerate(np.arange(nscans) / nvol):
scanstart = int((SCANONSET + trial * TR + (i % nvol) * TA) / dt)
scanidx = scanstart + np.arange(int(TA / dt))
timeline2[scanidx] = np.max(timeline)
reg.insert(i, np.mean(timeline[scanidx]) * reg_scale)
if isdefined(self.inputs.use_temporal_deriv) and \
self.inputs.use_temporal_deriv:
regderiv.insert(i, np.mean(timederiv[scanidx]) * reg_scale)
if isdefined(self.inputs.use_temporal_deriv) and \
self.inputs.use_temporal_deriv:
iflogger.info('orthoganlizing derivative w.r.t. main regressor')
regderiv = orth(reg, regderiv)
if bplot:
plt.subplot(4, 1, 3)
plt.plot(times, timeline2)
plt.subplot(4, 1, 4)
plt.bar(np.arange(len(reg)), reg, width=0.5)
plt.savefig('sparse.png')
plt.savefig('sparse.svg')
if regderiv:
return [reg, regderiv]
else:
return reg
def iszero(self):
""" Return True if the element is zero."""
return int(_C.GT_ELEM_is_zero(self.elem)) == 1
def merge_lines(lines, listpt, P):
thresh = copy.deepcopy(P["sgmnt_threshold"]['value'])
img_size = copy.deepcopy(P["img_size"])
# lines format: y1, x1, y2, x2, length, slope, alpha, index, start_pt, end_pt
# listpt = squeeze_arr(listpt)
# All lines that can be merged. Merging lines
# will group them together and then add the grouping to the list
out = [[n] for n in range(0, lines.shape[0])]
new_lp = listpt.tolist()
# Get unique start and end points. These are what we check
unique_pts = sort(unique(lines[:, 8:10]))
for index, ptx in enumerate(unique_pts):
# Test each combination of lines with this
# point to see which ones we can merge.
# Formula is combinations w/o repetitions (choose 2)
pairs = list(combinations(list(where(lines == ptx)[0]), 2))
# print(pairs)
# Go to next iteration if there's no combinations
if not pairs:
continue
for i, curr_pair in enumerate(pairs):
pt1, pt2, alph1, alph2, temp1, temp2 = relevant_lines(
curr_pair, lines)
# Check that the lines are within the threshold and not coincident
if abs(alph1 - alph2) > thresh or compare(temp1, temp2):
continue
# Get the unique start and end points of the two lines
lind1, lind2 = sort([
int(i) for i in list(
filter(lambda e: e not in [ptx], chain(temp1 + temp2)))
])
y1, x1 = unravel_index([lind1], img_size, order='F')
y2, x2 = unravel_index([lind2], img_size, order='F')
# print('y1', y1, 'x1', x1, 'y2', y2, 'x2', x2)
# print(unravel_index([lind1], img_size, order='F'))
slope, line_len, alpha = math_stuff(x1, y1, x2, y2)
# print('slope', slope)
# Intersection point is in the middle of the new line
if min(alph1, alph2) <= alpha <= max(alph1, alph2):
lines = delete(lines, max(pt1, pt2), axis=0)
lines = delete(lines, min(pt1, pt2), axis=0)
val1 = out[pt1]
val2 = out[pt2]
del out[max(pt1, pt2)]
del out[min(pt1, pt2)]
# Update both lists to reflect the addition of the merged line.
lines = append(lines, [[
int(y1),
int(x1) + 1,
int(y2),
int(x2) + 1, line_len, slope, alpha, 0, lind1, lind2
]],
axis=0)
out.append([val1, val2])
# listpt = merge_listpoints(listpt, pt1, pt2, lind1, lind2)
listpt = merge_lp(new_lp, pt1, pt2, lind1, lind2)
# Merged lines, so don't check the other pairs
break
else:
continue
return lines, array(listpt), array(out)
def rsaEncrypt(text, pubKey, modulus):
text = text[::-1]
rs = pow(int(binascii.hexlify(text), 16), int(pubKey, 16)) % int(modulus, 16)
return format(rs, 'x').zfill(256)
def check_input_args(input_filename=None,
output_filename=None,
input_part_size=None,
is_key_gen=None,
encrypt_mode=None,
key_file=None,
version_no=None,
print_arg_str=None,
print_encrypt_arg_str=None,
output_dir=None):
global version, is_encrypt_data, input_size, key_gen
version = version_no
is_encrypt_data = encrypt_mode
key_gen = is_key_gen
input_size = input_part_size
if not output_dir == os.getcwd() and (key_file
and os.path.isabs(key_file)):
sys.exit(
"Error. Cannot provide --outdir argument as --keyfile is absolute path."
)
if not os.path.isdir(output_dir):
distutils.dir_util.mkpath(output_dir)
if is_encrypt_data.lower() == 'true':
is_encrypt_data = True
elif is_encrypt_data.lower() == 'false':
is_encrypt_data = False
if version == 'v1':
version = Page.VERSION1
elif version == 'v2':
version = Page.VERSION2
if key_gen.lower() == 'true':
key_gen = True
elif key_gen.lower() == 'false':
key_gen = False
if key_gen:
if all(arg is not None
for arg in [input_filename, output_filename, input_size]):
if not is_encrypt_data:
sys.exit("--encrypt argument is missing or set to false.")
elif any(arg is not None
for arg in [input_filename, output_filename, input_size]):
sys.exit(print_arg_str)
else:
if not (input_filename and output_filename and input_size):
sys.exit(print_arg_str)
if is_encrypt_data and not key_gen and not key_file:
sys.exit(print_encrypt_arg_str)
if not is_encrypt_data and key_file:
sys.exit(
"Invalid. Cannot give --keyfile as --encrypt is set to false.")
if key_file:
key_file_name, key_file_ext = os.path.splitext(key_file)
if key_file_ext:
if not key_file_ext == '.bin':
sys.exit(
"--keyfile argument can be a filename with no extension or .bin extension only"
)
# If only one of the arguments - input_filename, output_filename, input_size is given
if ((any(arg is None for arg in [input_filename, output_filename, input_size])) is True) and \
((all(arg is None for arg in [input_filename, output_filename, input_size])) is False):
sys.exit(print_arg_str)
if input_size:
# Set size
input_size = int(input_size, 0)
if input_size % 4096 != 0:
sys.exit("Size of partition must be multiple of 4096")
# Update size as a page needs to be reserved of size 4KB
input_size = input_size - Page.PAGE_PARAMS["max_size"]
if input_size < (2 * Page.PAGE_PARAMS["max_size"]):
sys.exit("Minimum NVS partition size needed is 0x3000 bytes.")
def count(self, elem):
"""Count the number of times elem appears in the count."""
if not self.step:
return _coconut.float("inf") if elem == self.start else 0
return int(elem in self)
