def FindBootKernels(cls):
Tools.Print("Scanning " + config.kernelDirectory + " ...")
# Check to see if our boot directory exists before starting
if not os.path.exists(config.kernelDirectory):
Tools.Print("The " + config.kernelDirectory +
" directory doesn't exist. Creating ...")
os.mkdir(config.kernelDirectory)
if os.path.exists(config.kernelDirectory):
Tools.Warn("Please place your kernels inside " +
config.kernelDirectory + "/<version>, configure " +
ConfigLoader.GetConfigFilePath() +
", and then re-run the program. \n\nExample:\n\n" +
config.kernelDirectory +
"/3.12.12-KS.01/{vmlinuz, initrd}")
quit(1)
else:
Tools.Fail(config.kernelDirectory + " directory doesn't exist")
cmd = 'ls ' + config.kernelDirectory
results = check_output(["ls", config.kernelDirectory],
universal_newlines=True).strip()
# Add kernels to out kernel set
if results:
for i in results.split("\n"):
cls._bootKernels.append(i)
else:
Tools.Fail("No kernels found in " + config.kernelDirectory +
". A directory for each kernel you want must exist " +
"in that location.\n\nExample:\n\n" +
config.kernelDirectory + "/3.13.5-KS.01/\n" +
config.kernelDirectory + "/3.14.27-KS.01/\n")
def CreateOutputDirectory(cls, vParentDirectory):
if not os.path.exists(vParentDirectory):
os.makedirs(vParentDirectory)
if not os.path.exists(vParentDirectory):
Tools.Fail("Unable to create the " + vParentDirectory +
" directory ...")
def PrintCommonKernels(cls):
Tools.Print("Common kernels detected:")
for kernel in range(len(cls._commonKernels)):
print(
str(kernel + 1) + ". " + cls._commonKernels[kernel][0] +
" - " + cls._commonKernels[kernel][1])
def PrintKernelsInConfig(cls):
Tools.Print("Kernels detected in " + ConfigLoader.GetConfigFilePath() +
":")
for i in range(len(cls._configKernelLabel)):
print(
str(i + 1) + ". " + cls._configKernelLabel[i] + " - " +
cls._configKernelVersion[i])
def doTheRealJobNow(self):
""" Main Method """
tools = Tools(self.config, self.hostname, self.debug)
# LockFile Status erfragen, falls es noch nicht existiert - 1
lock_status = tools.getLockFilestatus()
if self.debug:
lock_status = 4
lock_status = -1
# kein Lock File > Rename Computer and Reboot
if lock_status == -1:
tools.Rename("Rename.ps1")
# tools.setLockFileStatus(1, "Host Renamed")
# das Restart File kann man nicht löschen, da es während des Reboots aktiv sein muss!
# tools.Restart()
"""
def __init__(cls):
# Get the fstab values for /boot immediately
cls.ScanFstab()
# boot drive
cls._bootDriveField = cls._fstabValues[0]
cls._bootDrive = Tools.MapIdentifierToDrive(cls._bootDriveField)
# Detect the layout of the /boot drive
cls.DetectDriveLayout()
def FindKernelsInConfig(cls):
Tools.Print("Scanning " + ConfigLoader.GetConfigFilePath() + " ...")
for kernel in config.kernels:
cls._configKernelLabel.append(kernel[0])
cls._configKernelVersion.append(kernel[1])
cls._configKernelDefault.append(kernel[2])
cls._configKernelName.append(kernel[3])
cls._configKernelInitrdName.append(kernel[4])
cls._configKernelOptions.append(kernel[5])
def ScanFstab(cls):
cmd = 'cat /etc/fstab | grep /boot[[:blank:]] | awk \'{print $1, $2, $3, $4, $5, $6}\''
results = check_output(cmd, shell=True,
universal_newlines=True).strip()
if results:
# Split the /boot line so we can store it
splits = results.split(" ")
# Save fstab /boot drive info
for x in splits:
cls._fstabValues.append(x.strip())
else:
Tools.Fail("/boot line could not be found in /etc/fstab")
def StripHead(cls, vPath):
if vPath:
splinters = vPath.split("/")
srange = len(splinters)
if srange >= 3:
news = ""
for i in range(len(splinters))[2:]:
news = news + "/" + splinters[i]
return news
elif srange == 2:
# if two then that means we will be looking in that folder specifically for kernels
# so just return /
return "/"
else:
Tools.Fail("The value to strip is empty ...")
def FindDefaultKernel(cls):
alreadyFound = 0
defaultKernelIndex = 0
for i in range(len(cls._commonKernels)):
if cls._commonKernels[i][2] == 1:
if alreadyFound == 0:
alreadyFound = 1
defaultKernelIndex = i
else:
Tools.Warn(
"Multiple default kernels detected. The default kernel will most likely not be correct!"
)
return defaultKernelIndex
if alreadyFound != 0:
return defaultKernelIndex
else:
return -1
def WriteEntries(cls):
outputFile = ""
driveLayout = Scanner.GetDriveLayout()
isOutput = Tools.IsOutputSet()
if isOutput:
outputFile = Tools.GetOutputFile()
# Check to see if the directory for this file exists. If it doesn't
# create any directories leading up to the output file so that we don't
# get a "FileNotFoundError" later on
outputFileFullPath = os.path.abspath(outputFile)
outputFileParentDir = os.path.dirname(outputFileFullPath)
if not os.path.exists(outputFileParentDir):
cls.CreateOutputDirectory(outputFileParentDir)
elif not isOutput and config.bootloader == "grub2":
outputFile = "grub.cfg"
elif not isOutput and config.bootloader == "extlinux":
outputFile = "extlinux.conf"
# Check to see what's the bootloader before we start adding
# all the entries, depending the bootloader we can cleanly start
# adding generic information like default kernel, timeouts, etc
position = Scanner.FindDefaultKernel()
if position != -1:
if config.bootloader == "grub2":
Tools.Print("Generating GRUB 2 configuration ...")
bootdrive = Scanner.GetGrub2BootDrive()
if os.path.exists(outputFile):
if Tools.IsForceSet():
dossier = open(outputFile, "w")
else:
Tools.Fail("Target file: " + outputFile +
" already exists. Pass -f to overwrite.")
else:
dossier = open(outputFile, "w")
dossier.write("set timeout=" + str(config.timeout) + "\n")
dossier.write("set default=" + str(position) + "\n")
dossier.write("\n")
# Write the modules that need to be inserted depending
# drive style. For whole disk zfs, none will be returned
# since a person can partition the drive manually and use msdos,
# or they can let zfs format their drive automatically with
# gpt. This ambiguity will be the reason both grub modules will
# be inserted.
if driveLayout == "gpt":
dossier.write("insmod part_gpt\n")
elif driveLayout == "msdos":
dossier.write("insmod part_msdos\n")
elif driveLayout == "none":
dossier.write("insmod part_gpt\n")
dossier.write("insmod part_msdos\n")
if config.efi:
dossier.write("insmod efi_gop\n")
dossier.write("insmod efi_uga\n")
dossier.write("insmod fat\n")
if config.wholeDiskZfs:
dossier.write("insmod zfs\n")
if config.goodyBag:
for candy in config.goodyBag:
dossier.write("insmod " + candy + "\n")
if not config.wholeDiskZfs:
dossier.write("\nset root='" + bootdrive + "'\n")
dossier.write("\n")
dossier.close()
elif config.bootloader == "extlinux":
Tools.Print("Generating extlinux configuration ...")
# Gets the name of the default kernel
defaultKernelLabel = Scanner.GetKernel(position)[0]
if os.path.exists(outputFile):
if Tools.IsForceSet():
dossier = open(outputFile, "w")
else:
Tools.Fail("Target file: " + outputFile +
" already exists. Pass -f to overwrite.")
else:
dossier = open(outputFile, "w")
dossier.write("TIMEOUT " + str(int(config.timeout * 10)) +
"\n")
if not config.extlinuxAutoBoot:
dossier.write("UI " + config.extlinuxUi + "\n")
dossier.write("\n")
dossier.write("DEFAULT " + defaultKernelLabel + str(position) +
"\n\n")
dossier.write("MENU TITLE " + config.extlinuxMenuTitle + "\n")
dossier.write("MENU COLOR title " + config.extlinuxTitleColor +
"\n")
dossier.write("MENU COLOR border " +
config.extlinuxBorderColor + "\n")
dossier.write("MENU COLOR unsel " +
config.extlinuxUnselectedColor + "\n")
dossier.write("\n")
dossier.close()
else:
Tools.Fail("The bootloader defined in " +
ConfigLoader.GetConfigFilePath() +
" is not supported.")
else:
Tools.Fail("The default kernel entry in " +
ConfigLoader.GetConfigFilePath() +
" was not found in " + config.kernelDirectory)
# Add all our desired kernels
for kernel in Scanner.GetCommonKernels():
# Get the position so that we can create the labels correctly
position = Scanner.GetKernelIndexInCommonList(kernel)
Tools.Warn("Adding: " + kernel[0] + " - " + kernel[1])
cs = cls.StripHead(config.kernelDirectory)
kernelPath = cs + "/" + kernel[1]
# Depending the bootloader we have specified, generate
# its appropriate configuration.
if config.bootloader == "grub2":
# Open it in append mode since the header was previously
# created before.
dossier = open(outputFile, "a")
dossier.write("menuentry \"" + kernel[0] + " - " + kernel[1] +
"\" {\n")
if config.wholeDiskZfs:
dossier.write("\tlinux " + bootdrive + "/@" + kernelPath +
"/" + kernel[3] + " " + kernel[5] + "\n")
if config.useInitrd:
dossier.write("\tinitrd " + bootdrive + "/@" +
kernelPath + "/" + kernel[4] + "\n")
else:
dossier.write("\tlinux " + kernelPath + "/" + kernel[3] +
" " + kernel[5] + "\n")
if config.useInitrd:
dossier.write("\tinitrd " + kernelPath + "/" +
kernel[4] + "\n")
dossier.write("}\n\n")
dossier.close()
elif config.bootloader == "extlinux":
dossier = open(outputFile, "a")
dossier.write("LABEL " + kernel[0] + str(position) + "\n")
dossier.write("\tMENU LABEL " + kernel[0] + " - " + kernel[1] +
"\n")
dossier.write("\tLINUX " + kernelPath + "/" + kernel[3] + "\n")
if config.useInitrd:
dossier.write("\tINITRD " + kernelPath + "/" + kernel[4] +
"\n")
dossier.write("\tAPPEND " + kernel[5] + "\n")
dossier.write("\n")
dossier.close()
# Append anything else the user wants automatically added
if config.append and config.appendStuff:
Tools.Print("Appending additional information ...")
if config.bootloader == "grub2":
dossier = open(outputFile, "a")
dossier.write(config.appendStuff)
dossier.close()
elif config.bootloader == "extlinux":
dossier = open(outputFile, "a")
dossier.write(config.appendStuff)
dossier.close()
# Check to make sure that the file was created successfully.
# If so let the user know..
if os.path.isfile(outputFile):
Tools.Success("'" + outputFile + "' has been created!")
else:
Tools.Fail(
"Either the file couldn't be created or the specified bootloader isn't supported."
)
def PrintBootKernels(cls):
Tools.Print("Kernels detected in the " + config.kernelDirectory +
" directory:")
for kernel in cls._bootKernels:
Tools.Print(kernel)
def GetGrub2BootDrive(cls):
# If we are using 'whole disk zfs', then we won't have a /boot entry
# in /etc/fstab. So instead we will format the zfs_boot variable and
# return it ready to be used in grub2
if config.wholeDiskZfs:
match = re.search('(/[a-zA-Z0-9_/]+)', config.wholeDiskZfsBootPool)
if match:
return match.group()
Tools.Fail(
"Could not parse the 'wholeDiskZfsBootPool' variable correctly."
)
# Properly processes the boot drive field in order for us to get
# a value that we can properly parse for the grub.cfg.
# This is so that if the user is using UUIDs as a /boot entry in
# /etc/fstab, we can handle that situation correctly.
match = re.search('/dev/(.*)', cls._bootDrive)
if match:
# Possibilities:
# sd[a-z][0+]
# vd[a-z][0+]
# md[0+]
# mapper/vg-root
# vg/root
# --- Handle sdX or vdX drives ---
m1 = re.search('[s|v]d(\w+)', match.group(1))
if m1:
# Complete value, will be completed as function progresses
completedValue = "(hd"
# Process the letter part and convert it to a grub compatible format; a = (hd0)
alph = re.search('\w', m1.group(1))
if alph:
# Find the number in the alphabet of this letter
alphindex = cls.GetAlphabeticalIndex(alph.group(0))
# Add this number to the final string
completedValue = completedValue + str(alphindex)
# Process the number part of the drive
numberPartOfDrive = re.search('\d', m1.group(1))
if numberPartOfDrive:
# add layout
completedValue = completedValue + "," + cls._driveLayout
# add number part
completedValue = completedValue + numberPartOfDrive.group(
0)
# close the value and return it
completedValue = completedValue + ")"
return completedValue
# --- Handle md# ---
m1 = re.search('md(\d+)', match.group(1))
if m1:
return "(md/" + m1.group(1) + ")"
# --- LVM: mapper/<volume_group>-<logical_volume> ---
m1 = re.search('mapper/(\w.-\w+)', match.group(1))
if m1:
return "(lvm/" + m1.group(1) + ")"
# --- LVM: <volume_group>/<logical_volume> ---
m1 = re.search('(\w+)/(\w+)', match.group(1))
if m1:
return "(lvm/" + m1.group(1) + "-" + m1.group(2) + ")"
# We've failed :(
Tools.Fail("Unable to generate the boot drive entry.")
def get_detail(self, source_url):
print "开始抓取" + source_url
soup = Tools().usePhantomjsGetHtml(source_url)
shadow_frame = soup.select("div.shadow-frame")[0]
row_fluid = shadow_frame.select('.row-fluid')
# 缩略图
try:
thumb_img = row_fluid[1].select(
"#wx_pic img:nth-of-type(1)")[0].get('src')
except Exception as e:
print str(e) + "缩略图没有获取到"
thumb_img = ""
# 电影介绍
try:
plot_introduce = str(
row_fluid[1].select(".span7")[0]).split('<br/>')
new_plot_introduce = []
for text in plot_introduce:
re_h = re.compile('</?\w+[^>]*>')
s = re_h.sub('', text)
new_plot_introduce.append(s)
plot_introduce = '<br/>'.join(new_plot_introduce)
except Exception as e:
print str(e) + '没有电影简介'
plot_introduce = ""
# 剧情简介
try:
content_info = row_fluid[2].get_text(' ', strip=True)
except Exception as e:
print "没有简介内容"
content_info = ""
# 标签
try:
tags = row_fluid[3].select(".badge.badge-warning i")
tags_arr = []
for tag in tags:
tags_arr.append(tag.get_text(strip=True))
tags = '|'.join(tags_arr)
except Exception as e:
print "标签为空"
tags = ""
# 种子下载地址
try:
seeds_tr = soup.select("#bs-docs-download tr")
except Exception as e:
print e
return False
link_down_source = []
for seed_tr in seeds_tr:
seed_a_href = seed_tr.select(".bd-address a")[0].get('href')
seed_a_title = seed_tr.select(".bd-address a")[0].get_text(
strip=True)
seeds = seed_tr.select("td:nth-of-type(2) a")
if len(seeds) != 0:
seeds_arr = []
for seed in seeds:
link_url = seed.get('href')
socure_name = seed.get_text(strip=True)
if socure_name == "迅雷":
source_type = 'xunlei'
elif socure_name == '旋风':
source_type = 'xuanfeng'
elif socure_name == '小米':
source_type = 'xiaomi'
elif socure_name == '看看':
source_type = 'kankan'
down_socurce = {}
down_socurce['link_url'] = link_url
down_socurce['source_type'] = source_type
seeds_arr.append(down_socurce)
link_down_source.append({
'title': seed_a_title,
'link_url': seed_a_href,
'seeds': seeds_arr
})
else:
# 百度云盘
try:
passwd = seed_tr.select(".bd-address span")[0].get_text(
strip=True)
link_down_source.append({
'title': seed_a_title,
'link_url': seed_a_href,
'passwd': passwd
})
except Exception as es:
print es
# seeds_title = seeds_tr.select("td.bd-address a")[0].get_text(strip=True)
# seeds = seeds_tr.select("td:nth-of-type(2) a")
# link_down_source = []
# for seed in seeds:
# link_url = seed.get('href')
# socure_name = seed.get_text(strip=True)
# if socure_name == "迅雷":
# source_type = 'xunlei'
# elif socure_name == '旋风':
# source_type = 'xuanfeng'
# elif socure_name == '小米':
# source_type = 'xiaomi'
# elif socure_name == '看看':
# source_type = 'kankan'
# down_socurce = {}
# down_socurce['title'] = seeds_title
# down_socurce['link_url'] = link_url
# down_socurce['source_type'] = source_type
# link_down_source.append(down_socurce)
# 百度云盘下载地址
detail_arr = {
'thumb_img': thumb_img,
'plot_introduce': plot_introduce,
'content_info': content_info,
'tags': tags,
'link_down_source': link_down_source
}
return detail_arr
def get_list(self, list_url):
soup = Tools().usePhantomjsGetHtml(list_url)
box_list = soup.select(".table-striped.shadow-frame tr td")
for td in box_list:
title = td.select("a:nth-of-type(2)>span")[0].get_text(
strip=True) # 标题
try:
douban_score = td.select(".list-douban")[0].get_text(
strip=True) or 0 # 评分
except Exception as e:
douban_score = 0
print e
old_updatetime = td.select(".text-info.f-right")[0].get_text(
strip=True) # 时间
source = td.select("a:nth-of-type(2)>span")[0].get("href") # 资源页面
# 创建md5对象
m = hashlib.md5()
m.update(source)
encryption_url = m.hexdigest() # 加密资源页面
film_cate = self.film_cate # 整体分类 如 2.电视剧 和 1.电影
# 先检查下
old_data = self.moive_cols.find_one(
{'encryption_url': encryption_url})
if old_data == None:
detail_res = self.get_detail(source)
if detail_res == False:
print "版权要求"
continue
new_data = {
'title':
title,
'hot_num':
0,
'source':
source,
'douban_score':
douban_score,
'old_updatetime':
old_updatetime,
'encryption_url':
encryption_url,
'film_cate':
film_cate,
'thumb_img':
detail_res['thumb_img'],
'plot_introduce':
detail_res['plot_introduce'],
'content_info':
detail_res['content_info'],
'tags':
detail_res['tags'],
'link_down_source':
detail_res['link_down_source'],
'create_time':
time.strftime('%Y-%m-%d', time.localtime(time.time()))
}
# 组合成mongodb所需要的数据
self.moive_cols.save(new_data)
else:
pass
print source + "抓取ok"
detail_arr = {
'thumb_img': thumb_img,
'plot_introduce': plot_introduce,
'content_info': content_info,
'tags': tags,
'link_down_source': link_down_source
}
return detail_arr
if __name__ == '__main__':
# 最新电影爬取
db_film = DbFilmSplider(1)
# 获取初始的页数
soup = Tools().usePhantomjsGetHtml("http://www.bd-film.com/zx/index.htm")
page_nums = soup.select(".input-prepend.input-append option")
nums_arr = []
for page_num in page_nums:
num = page_num.get('value')
nums_arr.append(int(num))
start_page = 1
last_page = max(nums_arr)
date_time = time.strftime("%Y-%m-%d", time.localtime())
try:
with open('./tmp/' + date_time + '.txt', 'r') as f:
s = f.read()
start_page = s or start_page