def download_dataset(self, folder_path):
url = "http://cs231n.stanford.edu/tiny-imagenet-200.zip"
from models.utils import Utils
folder_path = Utils.download_file(folder_path, url=url)
return Utils.extract_zip_file(file_path=folder_path,
extract_path='data')
def _get_blog_single(self, par):
ut = Utils()
self._build_widgets_single()
o = self._obj
# if it is the single blog view and the user is an editor then load up the images for the image select box
if self._req.sesh().can_edit():
o['images'] = Image.all().order("-date").fetch(self._conf.IMAGES_VIEWER_COUNT)
content = Content.get_by_key_name(par[1])
if content:
o['data'] = self.copy_bits(content)
# versions for admin - perhaps optimise this out with a isAdmin clause
o['data'].otherversions = content.contentversion_set
ov = []
for c in content.contentversion_set:
c.nicetime = c.createtime.strftime('%e %b %Y - %H:%M:%S')
ov.append(c)
o['data'].otherversions = ov
# what to put in the meta description - needed for singe blog post only
if content.current.summary:
o['data'].metadesc = ut.strip_tags(content.current.summary)
else:
o['data'].metadesc = o['data'].title
# no paging controls - for a single blog
o['page'] = {}
else:
return self._req.notfound()
self._respond(path='blog-single', obj=self._obj)
def parse_lectures_list(self, course_id):
with self.session.get(course_url.substitute(course_id=course_id),
cookies=self.cookies,
verify=False) as course:
lxml = html.fromstring(html=course.content)
urls = [_url.attrib['href'] for _url in lxml.find_class('item')]
lids = [Utils.get_id_from_url(_url) for _url in urls]
titles = [
Utils.flush(_title.text)
for _title in lxml.find_class('lecture-name')
]
wids = [self.parse_wistia_id(course_id, _lid) for _lid in lids]
durations = [
_dur.attrib['aria-valuemax']
for _dur in lxml.find_class('w-player-wrapper')
]
lectures = [
Lecture(_id=_id,
title=_title,
source=_wid,
duration=_dur,
path=_url) for _id, _title, _wid, _dur, _url in zip(
lids, titles, wids, durations, urls)
]
if self.lectures_list_len < 1:
self.lectures_list_len = len(lectures)
return lectures
def _post_new_group(self, par):
ut = Utils()
key = ut.cleanUserName(self._req.par('cid'))
group = Group(key_name=key)
group.title = self._req.par('ctitle')
group.put()
return self._req.redirect(path=self._conf.BLOG + '/' + key)
def download_lecture_from_cdn(selfs, lecture, course_title):
filename = f"{lecture.title}.mp4"
course_dir = Path("mosh_courses").resolve() / course_title
file = requests.get(lecture.path)
file_size = int(file.headers['Content-Length'])
print(
download_lecture_from_course_template.substitute(
lecture=filename,
course=course_title,
directory=course_dir,
fileSize=file_size))
Utils.write_file(course_dir, filename, file.content)
file.close()
def _get_blog_single(self, par):
ut = Utils()
self._build_widgets_single()
o = self._obj
# if it is the single blog view and the user is an editor then load up the images for the image select box
if self._req.sesh().can_edit():
o['images'] = Image.all().order("-date").fetch(self._conf.IMAGES_VIEWER_COUNT)
content = Content.get_by_key_name(par[1])
if content:
if self._req.sesh().can_edit():
o['data'] = content.editing
else:
o['data'] = content.current
o['data'].ctype = content.ctype
o['data'].status = content.status
o['data'].otherversions = content.contentversion_set
ov = []
for c in content.contentversion_set:
c.nicetime = c.createtime.strftime('%e %b %Y - %H:%M:%S')
ov.append(c)
o['data'].otherversions = ov
o['data'].keyname = content.key().name()
o['data'].group = content.group
# what to put in the meta description
if content.current.summary:
o['data'].metadesc = ut.strip_tags(content.current.summary)
else:
o['data'].metadesc = o['data'].title
o['data'].nicedate = content.sortdate.strftime('%e %b %Y')
# use hard coded imagepath, or the mainimage's url
if not o['data'].imagepath:
if o['data'].mainimage:
o['data'].imagepath = o['data'].mainimage.serving_url
# no paging controls - for a single blog
o['page'] = {}
else:
return self._req.notfound()
self._respond(path='blog-single', obj=self._obj)
def as_dict(self):
return {
'id': self.id,
'token': self.token,
'expires': Utils.get_unix_timestamp(self.expires),
'user': self.user.as_dict()
}
def training_step(self, batch, batch_idx):
images, masks, _, _ = batch
images, masks = Utils.preprocessing(images, masks, self.WL, self.WW)
images, masks = Utils.do_train_augmentations(images, masks,
self.gaussian_noise_std,
self.device, self.ra,
self.rf)
y_hat = self(images)
# loss dim is [batch, 1, img_x, img_y]
# need to get rid of the second dimension so
# size matches with mask
loss = self.loss(y_hat[:, 0, :, :], masks)
# Logs
#tensorboard_logs = {'train_loss': loss}
return {'loss': loss} #, 'log': tensorboard_logs}
def download_lecture_from_wistia(self, lecture, course_title):
response = requests.get(
wistia_json_url_template.substitute(wistia_id=lecture.source))
json_response = json.loads(Utils.get_json_from_callback(response.text))
response.close()
media = json_response["media"]
ready_mp4_url = media["assets"][0]["url"]
filename = f"{lecture.title}.mp4"
course_dir = Path("mosh_courses").resolve() / course_title
file = requests.get(ready_mp4_url)
print(
download_lecture_from_course_template.substitute(
lecture=filename, course=course_title, directory=course_dir))
Utils.write_file(course_dir, filename, file.content)
file.close()
def plotConfusionMatrix(self):
""" Plot a confusion matrix. """
if self.debug: print('In plotConfusionMatrix')
utils = Utils(debug=True)
cnf_matrix = confusion_matrix(self.y_test, self.predictions)
plt.figure()
utils.plot_confusion_matrix(cnf_matrix, classes=['Pos Sendiment','Neg Sendiment'], title='Confusion matrix, without normalization')
plt.figure()
utils.plot_confusion_matrix(cnf_matrix, classes=['Pos Sendiment','Neg Sendiment'], normalize=True, title='Confusion matrix, with normalization')
if self.debug: utils.printTP_FP_TN_FN(cnf_matrix)
def _post_new_content(self, par):
user = User(self._req.sesh())
person = user.get_sesh_person()
ut = Utils()
key = ut.cleanUserName(self._req.par('cid'))
# create the content record, by unique key - !important - replaces existing key - warn user?
content = Content(key_name=key)
content.status = 'draft'
groupname = self._req.par('cgroup')
content.group = Group.get_by_key_name(groupname)
# create a new version
contentver = ContentVersion()
# some defaults
contentver.title = self._req.par('ctitle')
contentver.content = '<p>Placeholder for the body of your Article</p>'
contentver.summary = '<p>Placeholder for the article summary</p>'
contentver.person = person
# have to put to get a reference
contentver.put()
# upate versions on this content
content.current = contentver
content.editing = contentver
content.put()
# link to my parent - in fact shouldnt I have used parents (but then sharding wont be an issue for a few, even 100's of, articles)
contentver.mycontent = content
contentver.put()
# and redirect to the new content
return self._req.redirect(path=self._conf.BLOG + '/' + groupname + '/' + key)
def parse_lecture_from_course(self, course_id, lecture_id):
with self.session.get(
f"https://codewithmosh.com/courses/{course_id}/lectures/{lecture_id}",
cookies=self.cookies,
verify=False) as lecture:
soup = html.fromstring(html=lecture.text)
title = Utils.flush(soup.xpath("//h2")[1].text_content())
url = soup.xpath("//*[contains(@class,'download')]")
if len(url) < 1:
yield Lecture(_id=lecture_id, title=title)
yield Lecture(
_id=lecture_id,
title=title,
path=url[0].attrib['href'] if 'href' in url[0].attrib else '')
def test_step(self, batch, batch_nb):
images, masks, _, _ = batch
images, masks = Utils.preprocessing(images, masks, self.WL, self.WW)
y_hat = self(images)
# loss dim is [batch, 1, img_x, img_y]
# need to get rid of the second dimension so
# size matches with mask
loss = self.loss(y_hat[:, 0, :, :], masks)
# Logs
#tensorboard_logs = {'val_loss': loss}
return {'test_loss': loss} #, 'log': tensorboard_logs}
def calculate_total_duration(self):
duration = 0.0
for lecture in self.lectures:
duration += lecture.duration
return Utils.from_secs_to_time(duration)
def _generate_token():
return hashlib.md5(str(datetime.now()) +
"//" +
Utils.generate_random_string()).hexdigest()
def migrate(dom=None, msg=None):
assert isinstance(dom, libvirt.virDomain)
assert isinstance(msg, dict)
# https://rk4n.github.io/2016/08/10/qemu-post-copy-and-auto-converge-features/
flags = libvirt.VIR_MIGRATE_PERSIST_DEST | \
libvirt.VIR_MIGRATE_UNDEFINE_SOURCE | \
libvirt.VIR_MIGRATE_COMPRESSED | \
libvirt.VIR_MIGRATE_PEER2PEER | \
libvirt.VIR_MIGRATE_AUTO_CONVERGE
root = ET.fromstring(dom.XMLDesc())
if msg['storage_mode'] == StorageMode.local.value:
# 需要把磁盘存放路径加入到两边宿主机的存储池中
# 不然将会报 no storage pool with matching target path '/opt/Images' 错误
flags |= libvirt.VIR_MIGRATE_NON_SHARED_DISK
flags |= libvirt.VIR_MIGRATE_LIVE
if not dom.isActive():
err = u'非共享存储不支持离线迁移。'
log_emit.warn(err)
raise RuntimeError(
'Nonsupport offline migrate with storage of non sharing mode.'
)
ssh_client = Utils.ssh_client(hostname=msg['duri'].split('/')[2],
user='******')
for _disk in root.findall('devices/disk'):
_file_path = _disk.find('source').get('file')
disk_info = Storage.image_info_by_local(path=_file_path)
disk_size = disk_info['virtual-size']
stdin, stdout, stderr = ssh_client.exec_command(' '.join([
'qemu-img', 'create', '-f', 'qcow2', _file_path,
str(disk_size)
]))
for line in stdout:
log_emit.info(line)
for line in stderr:
log_emit.error(line)
elif msg['storage_mode'] in [
StorageMode.shared_mount.value, StorageMode.ceph.value,
StorageMode.glusterfs.value
]:
if dom.isActive():
flags |= libvirt.VIR_MIGRATE_LIVE
flags |= libvirt.VIR_MIGRATE_TUNNELLED
else:
flags |= libvirt.VIR_MIGRATE_OFFLINE
# duri like qemu+ssh://destination_host/system
if dom.migrateToURI(duri=msg['duri'], flags=flags) == 0:
if msg['storage_mode'] == StorageMode.local.value:
for _disk in root.findall('devices/disk'):
_file_path = _disk.find('source').get('file')
if _file_path is not None:
os.remove(_file_path)
else:
raise RuntimeError('Unknown storage mode.')
def __init__(self):
self.session = requests.Session()
self.cookies = Utils.load_cookies()
self.lectures_list_len = 0
self.courses_list_len = 0
