def removefiles(removefile,root):
'''remove files from the location
iterate removefile removing files from root
moving them to the location specified in the file
app/cube.0.3.0_v2963.apk, removed/app
'''
logging.debug('remove files start (removefile:root)(' + removefile + ':' + root)
path = os.path.expanduser(removefile)
with open(path,'r') as f:
for line in f:
cl = line.strip();
args = cl.split(',')
if cl[:1] <> '#' and len(cl) > 0:
try:
print 'Attempt Move ' + cl
source = os.path.join(root, args[0].strip())
dest = os.path.join('working/', args[1].strip())
#if os.path.exists(source):
if len(glob.glob(source)) > 0:
logging.debug('copy ' + source + ' ' + dest )
logging.debug('Remove ' + source)
CheckMakeFolders([dest])
os.system('sudo cp ' + source + ' ' + dest)
#shutil.copy(source, dest)
os.system('sudo rm ' + root + args[0].strip())
else:
logging.debug(source + ' does not exist')
except IOError as e:
#print 'error ' + root + args[0].strip()
logerror('kitchen_utils::removefiles ' ,e,0)
logging.debug('remove files END')
logging.debug('================')
def finalise_boot_recovery(image):
'''pack up boot.img or recovery'''
try:
loc = image.find('.')
folder = image[:loc]
pprint('=')
pprint('Finalising ' + image)
print 'folder= ' + folder
pprint('=')
#check_make_folder('working/' + folder)
touch_and_zip_boot_recovery(folder,image)
print 'os.rename ' ,
print os.path.join('working', image),os.path.join('working', image + '.old')
os.rename(os.path.join('working', image),os.path.join('working', image + '.old'))
print 'renamed'
#logging.debug('rkcrc -k working/custom' + image + '.gz ' + image)
#os.system('rkcrc -k working/custom' + image + '.gz working/' + image)
rkcrc('-k', 'working/custom' + image + '.gz', 'working/' + image)
pprint('=')
pprint(image + ' finalised')
pprint('=')
except Exception as e:
logerror('kitchen_utils::finalise_boot_recovery ',e,1)
def parse_html(self, message):
utils.printf('%s:解析起始页开始...' % self.provider)
conn = utils.init_db('mysql', 'aiaabook', 2)
result = []
stmt = 'insert ignore into book(book_name,url,pub_year,cover_url) Values(%s,%s,%s,%s)'
cnt = 0
for filename, fullname in utils.file_list(self.html_path):
with open(fullname, encoding='utf8') as f:
text = f.read()
try:
sel = Selector(text=text)
for liTag in sel.xpath('//li[@class="search-item clearfix"]'):
book_name = liTag.xpath(
'./div/h4/a/text()').extract_first().strip()
url = liTag.xpath('./div/h4/a/@href').extract_first()
pub_year = liTag.xpath(
'.//div[@class="search-item__data-group__field meta__date"]/text()'
).extract_first()
cover_url = liTag.xpath(
'./div/a/img/@src').extract_first().strip()
result.append((book_name, url, pub_year, cover_url))
utils.printf(len(result))
except:
exMsg = '* ' + traceback.format_exc()
print(exMsg)
utils.logerror(exMsg)
utils.logerror(fullname)
return
utils.parse_results_to_sql(conn, stmt, result)
cnt += len(result)
utils.printf(cnt)
conn.close()
utils.printf('%s:解析起始页完成...' % self.provider)
self.senddistributefinish('startdown_detail')
def parse_index(self, message):
try:
utils.printf('%s:解析索引页开始...' % self.provider)
conn = utils.init_db('mysql', 'hepengineeringjournal', 4)
self.sqlList.clear()
cur = conn.cursor()
for filename, fullname in utils.file_list(self.index_path):
with open(fullname, encoding='utf8') as f:
text = f.read()
dic = json.loads(text, encoding='utf-8')
gch = filename.replace('.json', '')
dicitem = dic['resultValue']
issn = dicitem['issnNm']
cnno = dicitem['cnNm']
sql = 'update journal set issn="%s",cnno="%s" where journal_id="%s"' % (
issn, cnno, gch)
cur.execute(sql)
conn.commit()
cur.close()
conn.close()
utils.printf('%s:解析索引页完成...' % self.provider)
# self.sendwork('down_cover')
self.senddistributefinish('get_issuelist')
except:
exMsg = '* ' + traceback.format_exc()
print(exMsg)
utils.logerror(exMsg)
def ResizeSystem():
import system
try:
syssizeH, syssizeI, syssizeM, syssizeG = getsizes(rominfo.rominfo.system.size)
## sysosetH, sysosetI, sysosetM, sysosetG = getsizes( rominfo.rominfo.system.offset)
size = long(raw_input('Enter new system size bytes ' + str(syssizeI) + ' :'))
if size != syssizeI:
size, newuseroffsethex = sizetoparamsize(size,verbose = 1)
pprint('Applying these changes will adjust the parameters and system.img files.')
apply = raw_input('Apply these changes Y/N:')
if apply in ('y','Y'):
Resizemtdblock(hex(size/512)[:-1],'system')
## rominfo.rominfo.system.size = hex(size/512)[:-1]
## rominfo.rominfo.user.offset = newuseroffsethex
## rominfo.rominfo.applyparameterchanges()
system.shrinksystem()
else:
print 'No Change'
dummy = raw_input('press enter to continue')
except Exception as e:
logerror('parameter::ResizeSystem ',e,1)
def getcurrentparameterdatalines():
try:
sdata = rominfo.rominfo.parameterfile
syssizeH, syssizeI, syssizeM, syssizeG = getsizes(rominfo.rominfo.system.size)
userdatasizeH, userdatasizeI, userdatasizeM, userdatasizeG = getsizes(rominfo.rominfo.userdata.size)
sdata = sdata + ' \n' + 'Current system size:'
sdata = sdata + '\n' + ' Hex as paramters :' + syssizeH
sdata = sdata + '\n' + ' bytes :' + str(syssizeI)
sdata = sdata + '\n' + ' Mbytes :' + str(syssizeM)
sdata = sdata + '\n'
sdata = sdata + ' \n' + 'Current userdata (app space) size:'
sdata = sdata + '\n' + ' Hex as paramters :' + userdatasizeH
sdata = sdata + '\n' + ' bytes :' + str(userdatasizeI)
sdata = sdata + '\n' + ' Mbytes :' + str(userdatasizeM)
sdata = sdata + '\n' + ' Gbytes :' + str(userdatasizeG)
rdata = sdata.replace('\n\n','\n')
while rdata != sdata:
sdata = rdata
rdata = sdata.replace('\n\n','\n')
except Exception as e:
logerror('parameter::getcurrentparameterdatalines ',e,1)
return rdata
def flash_parameters():
'''flash parameters to tab
create a signed version then flash as follows
rkflashtool w 0x0 0x20 working/parameters_crc
rkflashtool w 0x400 0x20 working/parameters_crc
rkflashtool w 0x800 0x20 working/parameters_crc
rkflashtool w 0xc00 0x20 working/parameters_crc
rkflashtool w 0x1000 0x20 working/parameters_crc
rkflashtool w 0x1400 0x20 working/parameters_crc
rkflashtool w 0x1800 0x20 working/parameters_crc
rkflashtool w 0x1c00 0x20 working/parameters_crc
'''
try:
os.remove('working/parameters_crc')
except:
pass
try:
path = 'rkcrc'
os.system(path + ' -p working/parameter working/parameters_crc')
flash_singleimage('working/parameters_crc', '0x0 0x20')
flash_singleimage_noprompt('working/parameters_crc', '0x400 0x20')
flash_singleimage_noprompt('working/parameters_crc', '0x800 0x20')
flash_singleimage_noprompt('working/parameters_crc', '0xc00 0x20')
flash_singleimage_noprompt('working/parameters_crc', '0x1000 0x20')
flash_singleimage_noprompt('working/parameters_crc', '0x1400 0x20')
flash_singleimage_noprompt('working/parameters_crc', '0x1800 0x20')
flash_singleimage_noprompt('working/parameters_crc', '0x1c00 0x20')
os.remove('working/parameters_crc')
raw_input('press enter to continue')
except Exception as e:
logerror('flash::flash_parameters ', e, 1)
def down_index(self, message):
try:
journalname = message[0]
year = message[1]
fname = self.index_path + '/' + journalname + '_' + str(
year) + '.html'
utils.printf('开始下载 %s' % fname)
if os.path.exists(fname):
self.senddistributefinish('process_index')
return
feature = 'issue-month-detail'
url = 'http://{}.sciencemag.org/content/by/year/{}'.format(
journalname, str(year))
resp = self.gethtml(url, feature)
if not resp:
self.sendwork('down_index', (journalname, year))
return
with open(fname, mode='w', encoding='utf8') as f:
f.write(resp.content.decode('utf8'))
utils.printf('下载 %s 成功' % fname)
self.senddistributefinish('process_index')
except:
exMsg = '* ' + traceback.format_exc()
print(exMsg)
utils.logerror(exMsg)
def getcurrentparameterdatalines():
try:
sdata = rominfo.rominfo.parameterfile
syssizeH, syssizeI, syssizeM, syssizeG = getsizes(
rominfo.rominfo.system.size)
userdatasizeH, userdatasizeI, userdatasizeM, userdatasizeG = getsizes(
rominfo.rominfo.userdata.size)
sdata = sdata + ' \n' + 'Current system size:'
sdata = sdata + '\n' + ' Hex as paramters :' + syssizeH
sdata = sdata + '\n' + ' bytes :' + str(syssizeI)
sdata = sdata + '\n' + ' Mbytes :' + str(syssizeM)
sdata = sdata + '\n'
sdata = sdata + ' \n' + 'Current userdata (app space) size:'
sdata = sdata + '\n' + ' Hex as paramters :' + userdatasizeH
sdata = sdata + '\n' + ' bytes :' + str(userdatasizeI)
sdata = sdata + '\n' + ' Mbytes :' + str(userdatasizeM)
sdata = sdata + '\n' + ' Gbytes :' + str(userdatasizeG)
rdata = sdata.replace('\n\n', '\n')
while rdata != sdata:
sdata = rdata
rdata = sdata.replace('\n\n', '\n')
except Exception as e:
logerror('parameter::getcurrentparameterdatalines ', e, 1)
return rdata
def parameter_menu():
'''display parameter menu and process results'''
try:
my_menu = dict([
('a1', 'read parameters'),
('b2', 'display parameters'),
('c3', 'Edit Parameters'),
#('e5', 'finalise recovery'),
#('g=', '='),
#('h3', 'Custom remove'),
#('j4', 'Custom deploy'),
('t=', '='),
('vm', 'main menu'),
('z=', '=')
])
choice = mymenu(my_menu, 'Enter selection :', checkvalid=True)
if choice in ('1'):
parse_parameter()
elif choice in ('2'):
display_params()
elif choice in ('3'):
editparameter_menu()
elif choice in ('rR'):
repairparams()
else:
pass
if choice not in ('m', 'M'):
parameter_menu()
except Exception as e:
logerror('parameter::parameter_menu ', e, 1)
def saveromdata():
'''save the rom data to the rominfo
'''
try:
parse_parameter()
except Exception as e:
logerror('rkmainmenu::saveromdata ',e,1)
def unpackROM():
'''unpack the collected ROM image
'''
try:
tcwd = os.getcwd()
os.chdir(KitchenConfig.KitchenConfig.SourceROMUnpackedLoc())
image = os.path.join('../' , rominfo.rominfo.romimgfilename)
if image.find(' ') > 0:
image = "'" + image + "'"
logging.debug('rkunpack ' + image)
rkunpack(image)
#now fix bad parameter file if ends (use rather than (user)
testcorrectparameter()
#os.system( 'rkunpack ' + image )
os.chdir(tcwd)
except Exception as e:
os.chdir(tcwd)
logerror('rkmainmenu::getROMFile rkunpack fails ',e,0 )
#now if this does not work try rkunpack
try:
os.system( 'unpack_all.sh '
+ os.path.join(KitchenConfig.KitchenConfig.SourceROMLoc(), rominfo.rominfo.romimgfilename) + ' '
+ KitchenConfig.KitchenConfig.SourceROMUnpackedLoc())
except Exception as e:
logerror('rkmainmenu::unpackROM unpack_all.sh',e,1)
def makeROMkits():
'''make ROM kits based on current templates
This is a run once process
'''
try:
src = os.path.join(KitchenConfig.KitchenConfig.KitchenPath, 'ROMtemplates/')
dst = 'ROMKits/'
pprint ('Copying from ' + src + ' to ' + dst)
shutil.copytree(src,dst)
pprint ('Adding images to ROM Kits')
pcfilesrc = os.path.join(KitchenConfig.KitchenConfig.KitchenPath,'processcontrol/populateROMkits')
copyfilesworker(pcfilesrc,'',0,verbose = 1)
src = 'CWMROMKit.zip'
tgt = os.path.join(dst,rominfo.rominfo.romname.strip() + '_' + src)
src = os.path.join(dst,src)
os.system('mv ' + src + ' ' + tgt)
src = 'ROMKit.zip'
tgt = os.path.join(dst,rominfo.rominfo.romname.strip() + '_' + src)
src = os.path.join(dst,src)
os.system('mv ' + src + ' ' + tgt)
except Exception as e:
logerror('rkmainmenu::makeROMkits ',e,1)
def unpackBootRecoveryfile(thefile):
try:
go_on = 0
with open(thefile,'r') as f:
check = f.read(8)
if check == 'ANDROID!':
go_on=1
kernel = makeimage('Kernel',f)
ramdisk = makeimage('Ramdisk',f)
sramdisk = makeimage('Second Ramdisk',f)
baseaddress = getint(f.read(4))
pagesize = getint(f.read(4))
kernel.setpagesize(pagesize)
kernel.Offset = pagesize
ramdisk.Offset = kernel.NextPageStart()
sramdisk.Offset = ramdisk.NextPageStart()
if go_on ==1:
kernel.writeimage(thefile,thefile + '-kernel')
ramdisk.writeimage(thefile,thefile + '-ramdisk.gz')
if sramdisk.Size != 0:
sramdisk.writeimage(thefile,thefile + '-sramdisk.gz')
except Exception as e:
logerror('kitchen_utils::unpackBootRecoveryfile ',e,1)
def parsemtdblocks(CMDLINE,parsemtdonly = 0):
'''parse image data
'''
logging.debug('rominfo::parsemtdblocks CMDLINE = ' + CMDLINE)
try:
rominfo.mtdblocks = []
cmdlinedata = CMDLINE.split(':')
for i,v in enumerate(cmdlinedata):
if v.find('(cache)') > 0 :
idat = v
break
##parse image data
for i,v in enumerate(idat.split(',')):
tpos = v.find('@')
bpos = v.find('(')
size = v[:tpos]
offset = v[tpos+1:bpos]
image = v[bpos+1:].strip(')')
logging.debug('rominfo::parsemtdblocks image = ' + image)
if parsemtdonly != 1:
rominfo.setimage(image,offset,size)
rominfo.mtdblocks.append(image)
logging.debug('rominfo::parsemtdblocks image appended')
except Exception as e:
logerror('rominfo::parsemtdblocks ' ,e ,1)
def removefiles(removefile,root):
'''remove files from the location
iterate removefile removing files from root
moving them to the location specified in the file
app/cube.0.3.0_v2963.apk, removed/app
'''
logging.debug('remove files start (removefile:root)(' + removefile + ':' + root)
path = os.path.expanduser(removefile)
with open(path,'r') as f:
for line in f:
cl = line.strip();
args = cl.split(',')
if cl[:1] <> '#' and len(cl) > 0:
try:
print 'Attempt Move ' + cl
source = os.path.join(root, args[0].strip())
dest = os.path.join('working/', args[1].strip())
#if os.path.exists(source):
if len(glob.glob(source)) > 0:
logging.debug('copy ' + source + ' ' + dest )
logging.debug('Remove ' + source)
CheckMakeFolders([dest])
os.system('sudo cp ' + source + ' ' + dest)
#shutil.copy(source, dest)
os.system('sudo rm ' + root + args[0].strip())
else:
logging.debug(source + ' does not exist')
except IOError as e:
#print 'error ' + root + args[0].strip()
logerror('kitchen_utils::removefiles ' ,e,0)
logging.debug('remove files END')
logging.debug('================')
def applyparameterchanges():
'''Update the persisted parameter file with the current values and regenerate system.img and parameter files
'''
try:
logging.info('rominfo::applyparameterchanges start current parameter')
f = rominfo.parameterfile
logging.info('\n\n' + f + '\n\n')
#get all the mtdblocks here
kpendloc = f.find(',', f.find('(kpanic)'))+1
sysendloc = f.find(',', f.find('(system)'))+1
userendloc = f.find('(user)')+6
npf = f[:kpendloc]
npf = npf + rominfo.submtdsize(f[kpendloc:sysendloc],rominfo.system.size)
npf = npf + rominfo.submtdoffset(f[sysendloc:userendloc],rominfo.user.offset)
npf = npf + f[userendloc:]
rominfo.parameterfile = npf
logging.info('rominfo::applyparameterchanges update parameter')
f = rominfo.parameterfile
logging.info('\n\n' + f + '\n\n')
logging.info('rominfo::applyparameterchanges writing parameter file')
rominfo.writeparameter(npf)
logging.info('rominfo::applyparameterchanges storeparameterfile')
#rominfo.storeparameterfile('working/parameter')
logging.info('rominfo::applyparameterchanges end')
except Exception as e:
logerror('rominfo::applyparameterchanges ' ,e ,1)
def parse_indexlist(self, message):
try:
utils.printf('%s:解析期索引页开始...' % self.provider)
conn = utils.init_db('mysql', 'bioonejournal')
self.sqlList.clear()
cnt = 0
cur = conn.cursor()
path = '%s/%s' % (self.datepath, 'indexlist')
for filename, fullname in utils.file_list(path):
with open(fullname, encoding='utf8') as f:
text = f.read()
soup = BeautifulSoup(text, 'lxml')
aTags = soup.find_all('a', class_='IssueByYearInnerText')
for aTag in aTags:
url = aTag.get('href').replace('https://bioone.org', '')
self.sqlList.append(
"insert ignore into issuelist(url,year) Values('%s','%s')"
% (url, url.split('/')[-1]))
cnt += len(self.sqlList)
for sql in self.sqlList:
cur.execute(sql)
conn.commit()
self.sqlList.clear()
utils.printf(cnt)
cur.close()
conn.close()
utils.printf('%s:解析索引页完成...' % self.provider)
# self.sendwork('down_cover')
self.senddistributefinish('startdown_index')
except:
exMsg = '* ' + traceback.format_exc()
print(exMsg)
utils.logerror(exMsg)
def validatemtdblocks():
try:
ri = rominfo
mtdblocks = ri.mtdblocks
rollingoffset = 0
invalid = 0
errors = ''
#logging.info(mtdblocks)
for i,v in enumerate(mtdblocks):
offset = 0
size = 0
imgsize = 0
if v == 'use':
v = 'user'
logging.info(v)
size, offset, imgsize = ri.getmtddata(v,ri)
if size != '-':
ioffset = int(offset,16)
isize= int(size,16)
if ioffset < rollingoffset:
invalid = 1
errors += 'invalid offset {}\n'.format(v)
rominfo.errorblocks.append([v,'offset'])
rollingoffset += isize
isize =isize * 512
if imgsize > isize:
invalid = 1
errors += 'invalid {}.img, image larger than allocated space\n'.format(v)
rominfo.errorblocks.append([v,'size'])
except Exception as e:
logerror('rominfo::validatemtdblocks ' ,e ,1)
return invalid, errors
def Resizemtdblock(blocksize,tgtblock):
'''resize tgtblock to blocksize and adjust all following offsets
'''
try:
ri = rominfo.rominfo
mtdblocks = ri.mtdblocks
found = False
trailingblocks = []
#populate trailingblocks with our mtdblocks from our target onwards
for block in mtdblocks:
if block == tgtblock:
found = True
if found:
trailingblocks.append(block)
for block in trailingblocks:
#adjust the size of our target and the offsets of the following blocks
if block == tgtblock:
newoffset, pfile = subsize(ri.parameterfile,tgtblock,blocksize)
else:
newoffset, pfile = suboffset(pfile,block,newoffset)
ri.writeparameter(pfile)
except Exception as e:
logerror('parameter::Resizemtdblock ',e,1)
def StartKitchen():
'''Set basic config and logging options, reads complete config and start the kitchen
The actual Kitchen Config and if present the rominfo in the current workspace will be read, required folders as per config will be created, and then the rkmainmenu will be displayed.
'''
if GetCWD() == '/home/brian/rkKitchen':
os.chdir('/home/brian/Desktop/ROMS/cube/u30gt2/2.06_official')
print GetCWD
logmode = logging.INFO
writemode = 'a'
filename = 'kitchen.log'
for arg in sys.argv:
if arg == 'debug':
logmode = logging.DEBUG
elif arg == 'overwrite':
writemode = 'w'
logging.basicConfig(
filename=filename,
filemode=writemode,
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logmode)
try:
logging.info(
'===================================================================================='
)
logging.info('Start FreakKitchen')
logging.info(sys.version)
kc = KitchenConfig.KitchenConfig
kc()
kc.cwd = GetCWD()
print 'use py crc = {}'.format(kc.usepycrc)
ri = rominfo.rominfo
ri(KitchenConfig.KitchenConfig.ROMInfoLoc())
CheckMakeFolders(kc.KitchenFolders())
rkmainmenu()
kc.Pickle()
ri.Pickle()
except Exception as e:
logerror('FreakTabKitchen::StartKitchen ', e, 1)
finally:
pprint('=')
pprint('Thank you for using 900supersport' 's FreakTab RK Kitchen')
pprint('')
pprint('For support or even just to say thanks see')
pprint('=')
pprint(
'http://www.freaktab.com/showthread.php?8042-FreakTab-RK-ROM-Kitchen-by-900supersport-v2'
)
print
def parameter_menu():
'''display parameter menu and process results'''
try:
my_menu = dict([
('a1', 'read parameters'),
('b2', 'display parameters'),
('c3', 'Edit Parameters'),
#('e5', 'finalise recovery'),
#('g=', '='),
#('h3', 'Custom remove'),
#('j4', 'Custom deploy'),
('t=', '='),
('vm', 'main menu'),
('z=', '=')
])
choice = mymenu(my_menu,'Enter selection :',checkvalid = True)
if choice in ('1'):
parse_parameter()
elif choice in ('2'):
display_params()
elif choice in ('3'):
editparameter_menu()
elif choice in ('rR'):
repairparams()
else:
pass
if choice not in ('m','M'):
parameter_menu()
except Exception as e:
logerror('parameter::parameter_menu ',e,1)
def browse_system():
'''launch natilus and point at working'''
try:
kc = KitchenConfig.KitchenConfig
os.system('sudo ' + kc.browser + ' working/mntsystem')
except Exception as e:
logerror('system::browse_system ' , e, 1)
def parse_html(self):
utils.printf('%s:解析起始页开始...' % self.provider)
conn = utils.init_db('mysql', 'hepengineeringjournal', 4)
result = []
stmt = 'insert ignore into journal(journal_id,journal_name,cover_url) Values(%s,%s,%s)'
cnt = 0
for filename, fullname in utils.file_list(self.html_path):
with open(fullname, encoding='utf8') as f:
text = f.read()
try:
dic = json.loads(text, encoding='utf8')
for dicitem in dic['resultValue']:
dicitem = json.loads(dicitem, encoding='utf8')
gch = dicitem['id']
name = dicitem['name']
cover_url = dicitem['volumeImg']
if cover_url == '':
cover_url = dicitem['journalImg']
print(gch, name, cover_url)
result.append((gch, name, cover_url))
except:
exMsg = '* ' + traceback.format_exc()
print(exMsg)
utils.logerror(exMsg)
utils.parse_results_to_sql(conn, stmt, result)
cnt += len(result)
utils.printf(cnt)
conn.close()
utils.printf('%s:解析起始页完成...' % self.provider)
self.senddistributefinish('startdown_index')
def parsemtdblocks(CMDLINE, parsemtdonly=0):
'''parse image data
'''
logging.debug('rominfo::parsemtdblocks CMDLINE = ' + CMDLINE)
try:
rominfo.mtdblocks = []
cmdlinedata = CMDLINE.split(':')
for i, v in enumerate(cmdlinedata):
if v.find('(cache)') > 0:
idat = v
break
##parse image data
for i, v in enumerate(idat.split(',')):
tpos = v.find('@')
bpos = v.find('(')
size = v[:tpos]
offset = v[tpos + 1:bpos]
image = v[bpos + 1:].strip(')')
logging.debug('rominfo::parsemtdblocks image = ' + image)
if parsemtdonly != 1:
rominfo.setimage(image, offset, size)
rominfo.mtdblocks.append(image)
logging.debug('rominfo::parsemtdblocks image appended')
except Exception as e:
logerror('rominfo::parsemtdblocks ', e, 1)
def cleanworkspace():
try:
cont = 'Y'
rn = rominfo.rominfo.romname
rimg = rominfo.rominfo.romimgfilename
src = os.path.join('sources',rn,rimg)
dst = rimg
#retrieve original img file
if os.path.isfile(src):
os.system('mv ' + src + ' ' + dst)
else:
cont = raw_input('No ROM image found \n' + src + '\n\nContinue Y/N:')
#retrieve and archive any pulled images
if cont in ('Y', 'y'):
zipfolder('read','ReadImages.zip')
zipfolder('localdeploy','localdeploy.zip')
os.system('sudo rm -rf working')
shutil.rmtree('localdeploy')
shutil.rmtree('sources')
shutil.rmtree('read')
rominfo.rominfo.romname = 'un-initialised'
except Exception as e:
logerror('rkmainmenu::recursive_zip ',e,1)
def parse_index(self, message):
try:
utils.printf('%s:解析索引页开始...' % self.provider)
conn = utils.init_db('mysql', 'science')
result = []
stmt = 'insert ignore into issue(url,stat) Values(%s,%s)'
cnt = 0
for filename, fullname in utils.file_list(self.index_path):
urlf = '{}.sciencemag.org'.format(filename.split('_')[0])
with open(fullname, encoding='utf8') as f:
text = f.read()
soup = BeautifulSoup(text, 'lxml')
divTags = soup.find_all(
'div',
class_=
'highwire-cite highwire-cite-highwire-issue highwire-citation-jnl-sci-issue-archive clearfix'
)
for divTag in divTags:
url = urlf + divTag.a.get('href')
result.append((url, 0))
if utils.parse_results_to_sql(conn, stmt, result, 1000):
cnt += len(result)
result.clear()
utils.printf(cnt)
utils.parse_results_to_sql(conn, stmt, result)
cnt += len(result)
utils.printf(cnt)
conn.close()
utils.printf('%s:解析索引页完成...' % self.provider)
self.senddistributefinish('startdown_list')
except:
exMsg = '* ' + traceback.format_exc()
print(exMsg)
utils.logerror(exMsg)
def unpackboot_recovery(image):
'''unpack a boot.img or recovery.img'''
try:
loc = image.find('.')
folder = image[:loc]
filepath = os.path.join('working/', image)
with open(filepath,'rb') as f:
disc = f.read(4)
if disc == 'KRNL':
logging.info('kitchen_utils::unpack_recovery boot.img signed KRNL')
os.system('dd if=working/' + image + ' of=working/' + image + '-ramdisk.gz'
+ ' skip=8 bs=1 count=20000000')
else:
unpackBootRecoveryfile(filepath)
logging.debug('kitchen_utils::unpack_recovery :')
imagefolder = os.path.join('working', folder)
CheckMakeFolders([imagefolder])
os.chdir(imagefolder)
#protect against this being first sudo request
os.system('sudo ls')
os.system('sudo gunzip < ../' + image + '-ramdisk.gz '
+ '| sudo cpio -i --make-directories')
os.chdir('../..')
except Exception as e:
logerror('kitchen_utils::unpackboot_recovery ',e,1)
def startdown_list(self, message):
utils.printf('%s:开始下载列表页...' % self.provider)
if not self.list_path:
self.initpath()
self.refreshproxypool()
self.count = 0
conn = utils.init_db('mysql', 'science')
cur = conn.cursor()
cur.execute('select url,stat from issue where stat=0')
rows = cur.fetchall()
self.totalcount = len(rows)
if self.totalcount == 0:
if len(os.listdir(self.index_path)) == 0:
utils.logerror('%s:没有新的issue不需要更新' % self.provider)
utils.msg2weixin('%s:没有新的issue不需要更新' % self.provider)
else:
self.sendwork('parse_list')
for url, _ in rows:
fdir = self.list_path + '/' + url.split('.')[0]
if not os.path.exists(fdir):
os.makedirs(fdir)
fname = fdir + '/' + url.split('/')[-2] + '_' + url.split(
'/')[-1] + '.html'
url = 'http://' + url
self.sendwork('down_list', (url, fname))
def makeimage(iname,f):
try:
s = getint(f.read(4))
a = getint(f.read(4))
return part(iname,s,a)
except Exception as e:
logerror('kitchen_utils::makeimage ',e,1)
def saveromdata():
'''save the rom data to the rominfo
'''
try:
parse_parameter()
except Exception as e:
logerror('kitchen_utils::saveromdata ',e,1)
def startdown_list(self, message):
utils.printf('%s:开始下载列表页...' % self.provider)
if not self.list_path:
self.initpath()
self.refreshproxypool()
self.count = 0
conn = utils.init_db('mysql', 'apsjournal')
cur = conn.cursor()
current_year = time.strftime('%Y')
cur.execute(
"select url,stat from issue where stat=0 or year=%s or year=%s" %
(current_year, int(current_year) - 1))
rows = cur.fetchall()
self.totalcount = len(rows)
if self.totalcount == 0:
if len(os.listdir(self.list_path)) == 0:
utils.logerror('%s:没有新的issue不需要更新' % self.provider)
else:
self.sendwork('parse_list')
for url, _ in rows:
fdir = self.list_path + '/' + url.split('/')[-4]
if not os.path.exists(fdir):
os.makedirs(fdir)
fname = fdir + '/' + url.split('/')[-2] + '_' + url.split(
'/')[-1] + '.html'
self.sendwork('down_list', (url, fname))
def unpackBootRecoveryfile(thefile):
try:
go_on = 0
with open(thefile,'r') as f:
check = f.read(8)
if check == 'ANDROID!':
go_on=1
kernel = makeimage('Kernel',f)
ramdisk = makeimage('Ramdisk',f)
sramdisk = makeimage('Second Ramdisk',f)
baseaddress = getint(f.read(4))
pagesize = getint(f.read(4))
kernel.setpagesize(pagesize)
kernel.Offset = pagesize
ramdisk.Offset = kernel.NextPageStart()
sramdisk.Offset = ramdisk.NextPageStart()
if go_on ==1:
kernel.writeimage(thefile,thefile + '-kernel')
ramdisk.writeimage(thefile,thefile + '-ramdisk.gz')
if sramdisk.Size != 0:
sramdisk.writeimage(thefile,thefile + '-sramdisk.gz')
except Exception as e:
logerror('kitchen_utils::unpackBootRecoveryfile ',e,1)
def unpackboot_recovery(image):
'''unpack a boot.img or recovery.img'''
try:
loc = image.find('.')
folder = image[:loc]
filepath = os.path.join('working/', image)
with open(filepath,'rb') as f:
disc = f.read(4)
if disc == 'KRNL':
logging.info('kitchen_utils::unpack_recovery boot.img signed KRNL')
os.system('dd if=working/' + image + ' of=working/' + image + '-ramdisk.gz'
+ ' skip=8 bs=1 count=20000000')
else:
unpackBootRecoveryfile(filepath)
logging.debug('kitchen_utils::unpack_recovery :')
imagefolder = os.path.join('working', folder)
CheckMakeFolders([imagefolder])
os.chdir(imagefolder)
#protect against this being first sudo request
os.system('sudo ls')
os.system('sudo gunzip < ../' + image + '-ramdisk.gz '
+ '| sudo cpio -i --make-directories')
os.chdir('../..')
except Exception as e:
logerror('kitchen_utils::unpackboot_recovery ',e,1)
def makeimage(iname,f):
try:
s = getint(f.read(4))
a = getint(f.read(4))
return part(iname,s,a)
except Exception as e:
logerror('kitchen_utils::makeimage ',e,1)
def finalise_boot_recovery(image):
'''pack up boot.img or recovery'''
try:
loc = image.find('.')
folder = image[:loc]
pprint('=')
pprint('Finalising ' + image)
print 'folder= ' + folder
pprint('=')
#check_make_folder('working/' + folder)
touch_and_zip_boot_recovery(folder,image)
print 'os.rename ' ,
print os.path.join('working', image),os.path.join('working', image + '.old')
os.rename(os.path.join('working', image),os.path.join('working', image + '.old'))
print 'renamed'
#logging.debug('rkcrc -k working/custom' + image + '.gz ' + image)
#os.system('rkcrc -k working/custom' + image + '.gz working/' + image)
rkcrc('-k', 'working/custom' + image + '.gz', 'working/' + image)
pprint('=')
pprint(image + ' finalised')
pprint('=')
except Exception as e:
logerror('kitchen_utils::finalise_boot_recovery ',e,1)
def startdown_list(self, message):
utils.printf('%s:开始下载列表页...' % self.provider)
if not self.list_path:
self.initpath()
self.sqlList.clear()
self.refreshproxypool()
self.count = 0
conn = utils.init_db('mysql', 'cambridgejournal')
cur = conn.cursor()
cur.execute('select url,stat from issue where stat=0')
rows = cur.fetchall()
self.totalcount = len(rows)
if self.totalcount == 0:
if len(os.listdir(self.list_path)) == 0:
utils.logerror('%s:没有新的issue不需要更新' % self.provider)
else:
# self.sendwork('down_cover')
self.sendwork('parse_list')
return
self.refreshsession()
for url, _ in rows:
fdir = self.list_path + '/' + url.split('/')[-3]
if not os.path.exists(fdir):
os.makedirs(fdir)
flast = url.split('/')[-1]
if flast.find('?pageNum=') > 0:
flast = flast.split('?')[0] + '_' + flast.split('=')[-1]
fname = fdir + '/' + flast + '.html'
self.sendwork('down_list', (url, fname))
def header(checkvalid = False):
'''Display a generic header'''
try:
logging.debug('Start header')
kernelimage = rominfo.rominfo.kernelImageSize
print
print
pprint( '=')
pprint( 'FreakTab RK ROM Kitchen by 900supersport v2.0.1')
pprint( 'Brian Mahoney')
pprint( '1 Dec 2013')
pprint( 'www.freaktab.com')
pprint( '=')
romname = rominfo.rominfo.romname
pprint( 'Current ROM ' + romname)
pprint( 'CWD ' + KitchenConfig.KitchenConfig.cwd)
if kernelimage <> '':
pprint( 'Kernel image size:' + kernelimage)
pprint( '=')
if romname != 'un-initialised' and checkvalid:
ri = rominfo.rominfo
invalid, errors = ri.validatemtdblocks()
if invalid != 0:
pprint( 'Ivalid ROM',1)
for l in errors.split('\n'):
pprint(l, 1)
pprint(' ')
pprint('Press R to attempt repair',1)
pprint('=')
except Exception as e:
logerror('KitchenUI::header ',e,1)
logging.debug('End header')
def query_add(filepath, check, addition):
'''check the file specified in filepath for the string 'check'
and if not found add the text 'addition' to the end ofthe file
'''
try:
#print 'query_add start'
#print filepath
#print check
#print addition
check = check.strip("'")
check = check.strip('"')
check = check.strip(' ')
addition = addition.strip("'")
addition = addition.strip('"')
addition = addition.strip(' ')
with open(filepath,'r+') as f:
initrf = f.read()
found = initrf.find(check)
print('query_add::check: ' + check)
#print('query_add::found: {}'.format( found))
print('query_add::addition: ' + addition)
if found == -1:
f.write('\n')
f.write(addition.strip())
except Exception as e:
logerror('KitchenRunasroot::query_add ',e,1)
def startdown_list(self, message):
utils.printf('%s:开始下载列表页...' % self.provider)
if not self.list_path:
self.initpath()
self.refreshproxypool()
self.sqlList.clear()
self.count = 0
conn = utils.init_db('mysql', 'aiaajournal', 2)
cur = conn.cursor()
cur.execute('select url,stat from issue where stat=0')
rows = cur.fetchall()
self.totalcount = len(rows)
if self.totalcount == 0:
if len(os.listdir(self.list_path)) == 0:
utils.logerror('%s:没有新的issue不需要更新' % self.provider)
else:
self.sendwork('parse_list')
for url, _ in rows:
urlsp = url.split('/')
base_name = '%s_%s.html' % (urlsp[-2], urlsp[-1])
fdir = '%s/%s' % (self.list_path, urlsp[-3])
fname = '%s/%s' % (fdir, base_name)
if not os.path.exists(fdir):
os.makedirs(fdir)
self.sendwork('down_list', (url, fname))
def pprint(pstring,error = 0):
'''Pretty print the string
Basic pretty print, if pstring = then a line of =
othewise if it fits prepend '| ' then pad end with space ' |' to width
'''
try:
logging.debug('Start pprint: ' + pstring)
width = KitchenConfig.KitchenConfig.pwidth
logging.debug('KitchenUI::pprint width: ' + str(width))
FAIL = '\033[91m'
ENDC = '\033[0m'
COL = '\033[0;34;47m'
pstring=pstring.rstrip()
lpstring = len(pstring)
pad = width - 4 - lpstring
if pstring[:1] == '=' and lpstring == 1:
pstring = COL + '='*width + ENDC
elif pad > 0 :
if error == 1:
pstring = FAIL + pstring + COL
pstring = COL + '| ' + pstring + ' '*pad + ' |' + ENDC
print pstring
except Exception as e:
logerror('KitchenUI::pprint ',e,1)
logging.debug('end pprint')
def query_add(filepath, check, addition):
'''check the file specified in filepath for the string 'check'
and if not found add the text 'addition' to the end ofthe file
'''
try:
#print 'query_add start'
#print filepath
#print check
#print addition
check = check.strip("'")
check = check.strip('"')
check = check.strip(' ')
addition = addition.strip("'")
addition = addition.strip('"')
addition = addition.strip(' ')
with open(filepath, 'r+') as f:
initrf = f.read()
found = initrf.find(check)
print('query_add::check: ' + check)
#print('query_add::found: {}'.format( found))
print('query_add::addition: ' + addition)
if found == -1:
f.write('\n')
f.write(addition.replace(r'\t', '\t').replace(r'\n', '\n'))
except Exception as e:
logerror('KitchenRunasroot::query_add ', e, 1)
def parse_index(self, message):
try:
utils.printf('%s:解析索引页开始...' % self.provider)
conn = utils.init_db('mysql', 'aiaajournal', 2)
result = []
stmt = 'insert ignore into issue(url,stat) Values(%s,%s)'
cnt = 0
for filename, fullname in utils.file_list(self.index_path):
with open(fullname, encoding='utf8') as f:
text = f.read()
sel = Selector(text=text)
for aTag in sel.xpath('//a[@class="loi__issue__vol"]'):
url = aTag.xpath('./@href').extract_first()
if url.endswith('/0/0'):
continue
result.append(('https://arc.aiaa.org' + url, 0))
if utils.parse_results_to_sql(conn, stmt, result, 200):
cnt += len(result)
result.clear()
utils.printf(cnt)
utils.parse_results_to_sql(conn, stmt, result)
cnt += len(result)
utils.printf(cnt)
conn.close()
utils.printf('%s:解析索引页完成...' % self.provider)
self.senddistributefinish('startdown_list')
except:
exMsg = '* ' + traceback.format_exc()
print(exMsg)
utils.logerror(exMsg)
def saveromdata():
'''save the rom data to the rominfo
'''
try:
parse_parameter()
except Exception as e:
logerror('kitchen_utils::saveromdata ',e,1)
def startdown_list(self, message):
utils.printf('%s:开始下载列表页...' % self.provider)
if not self.list_path:
self.initpath()
self.refreshproxypool()
self.sqlList.clear()
self.count = 0
conn = utils.init_db('mysql', 'hepjournal', 4)
cur = conn.cursor()
cur.execute('select url,journal_id from issue where stat=0')
rows = cur.fetchall()
self.totalcount = len(rows)
if self.totalcount == 0:
if len(os.listdir(self.list_path)) == 0:
utils.logerror('%s:没有新的issue不需要更新' % self.provider)
else:
# self.sendwork('down_cover')
self.sendwork('parse_list')
for url, journal_id in rows:
fdir = self.list_path + '/' + journal_id
if not os.path.exists(fdir):
os.makedirs(fdir)
fname = fdir + '/' + journal_id + '_' + url.split(
'/')[-2] + '_' + url.split('/')[-1].replace('.shtml', '.html')
self.sendwork('down_list', (url, fname))
def gethtml(self, url, feature=None, coverflag=False):
try:
resp = self.session.get(url,
headers=self.headers,
timeout=20,
proxies=self.proxies)
if not coverflag:
if resp.content.decode('utf-8').find('Just a moment...') > 0:
utils.printf('Just a moment...')
if not self.refreshflag:
self.refreshflag = True
self.refreshsession()
else:
while self.refreshflag:
time.sleep(1)
resp = self.session.get(url,
headers=self.headers,
timeout=20,
proxies=self.proxies)
if resp.status_code != 200:
print('code !=200')
return False
if resp.content.decode('utf-8').find('</html>') < 0:
print('not endwith </html>')
return False
if feature:
if resp.content.decode('utf-8').find(feature) < 0:
print('can not find feature')
utils.logerror(url)
return False
except:
return False
return resp
def recoverymenu():
'''display recovery menu and process results'''
try:
my_menu = dict([
('a1', 'unpack recovery.img'),
('b2', 'browse recovery'),
#('c3', 'init.d support'),
('e5', 'finalise recovery'),
#('f6', 'Pack Kernel with fun_ CWM recovery'),
('g=', '='),
('h3', 'Custom remove'),
('j4', 'Custom deploy'),
('t=', '='),
('vm', 'main menu'),
('z=', '=')
])
choice = mymenu(my_menu,'Enter selection :')
if choice in ('1'):
unpackrecovery()
elif choice in ('2'):
browse_recovery()
elif choice in ('5'):
finalise_recovery()
#elif choice in ('6'):
# pack_CWM_recovery()
else:
pass
if choice not in ('m','M'):
recoverymenu()
except Exception as e:
logerror('recovery::recoverymenu ',e,1)
def flash_parameters():
'''flash parameters to tab
create a signed version then flash as follows
rkflashtool w 0x0 0x20 working/parameters_crc
rkflashtool w 0x400 0x20 working/parameters_crc
rkflashtool w 0x800 0x20 working/parameters_crc
rkflashtool w 0xc00 0x20 working/parameters_crc
rkflashtool w 0x1000 0x20 working/parameters_crc
rkflashtool w 0x1400 0x20 working/parameters_crc
rkflashtool w 0x1800 0x20 working/parameters_crc
rkflashtool w 0x1c00 0x20 working/parameters_crc
'''
try:
os.remove('working/parameters_crc')
except:
pass
try:
path ='rkcrc'
os.system(path + ' -p working/parameter working/parameters_crc')
flash_singleimage('working/parameters_crc','0x0 0x20')
flash_singleimage_noprompt('working/parameters_crc','0x400 0x20')
flash_singleimage_noprompt('working/parameters_crc','0x800 0x20')
flash_singleimage_noprompt('working/parameters_crc','0xc00 0x20')
flash_singleimage_noprompt('working/parameters_crc','0x1000 0x20')
flash_singleimage_noprompt('working/parameters_crc','0x1400 0x20')
flash_singleimage_noprompt('working/parameters_crc','0x1800 0x20')
flash_singleimage_noprompt('working/parameters_crc','0x1c00 0x20')
os.remove('working/parameters_crc')
raw_input('press enter to continue')
except Exception as e:
logerror('flash::flash_parameters ',e,1)
def parse_index(self, message):
workdir = message
try:
utils.printf('%s:解析索引页开始...' % self.provider)
conn = utils.init_db('mysql', 'apsjournal')
result = []
stmt = 'insert ignore into issue(url,year) Values(%s,%s) on DUPLICATE key UPDATE year=%s'
cnt = 0
for filename, fullname in utils.file_list(workdir):
with open(fullname, encoding='utf8') as f:
text = f.read()
soup = BeautifulSoup(text, 'lxml')
liTags = soup.select('div.volume-issue-list > ul > li')
for liTag in liTags:
yeartext = liTag.get_text().strip()
year = re.sub('.*?(\d{4}) \(.*?\)', r'\1', yeartext)
url = 'https://journals.aps.org' + liTag.b.a.get('href')
result.append((url, year, year))
if utils.parse_results_to_sql(conn, stmt, result, 1000):
cnt += len(result)
result.clear()
utils.printf(cnt)
utils.parse_results_to_sql(conn, stmt, result)
cnt += len(result)
utils.printf(cnt)
conn.close()
utils.printf('%s:解析索引页完成...' % self.provider)
self.senddistributefinish('startdown_list')
except:
exMsg = '* ' + traceback.format_exc()
print(exMsg)
utils.logerror(exMsg)
def __init__(self,name,size,offset):
try:
self.Name = name.strip()
self.Size = size
self.Offset = offset
part._pagesize = 0
except Exception as e:
logerror('kitchen_utils::part::__init__ ',e,1)
def browse_boot():
'''launch browser and point at working/boot'''
try:
cmd = 'sudo ' + KitchenConfig.KitchenConfig.browser + ' working/boot'
logging.debug('boot::browse_boot ' + cmd)
os.system(cmd)
except Exception as e:
logerror('boot::browse_boot ',e,1)
def browse(path):
'''launch natilus and point at working'''
try:
kc = KitchenConfig.KitchenConfig
os.system('sudo ' + kc.browser + ' ' + path)
except Exception as e:
logerror('kitchen_utils::browse ',e,1)
def brand_boot(openforreview):
'''brad boot.img'''
try:
spath = os.path.join(KitchenConfig.KitchenConfig.KitchenPath, 'processcontrol/bootdeploy')
deployfiles(spath
,'working/boot/'
,openforreview)
except Exception as e:
logerror('boot::brand_boot ',e,1)
def flash_reboot():
'''reboot the device'''
try:
s = 'sudo rkflashtool b '
print s
os.system(s)
except Exception as e:
logerror('flash::flash_reboot ',e,1)
def bootmenu():
'''display boot menu and process results'''
try:
my_menu = dict([
('a1', 'unpack boot.img'),
('b2', 'build prop edits'),
('c3', 'init.d support'),
('d4', 'brand boot.img'),
('e5', 'finalise boot'),
('g=', '='),
('h6', 'Custom remove'),
('j7', 'Custom deploy'),
('k8', 'Custom extend'),
('l9', 'Custom extend init.rk30board.rc'),
('p=', '='),
('qw', 'broWse'),
('t=', '='),
('vm', 'main menu'),
('z=', '=')
])
choice = mymenu(my_menu,'Enter selection :')
if choice in ('1'):
unpackboot()
elif choice in ('2'):
initrc_mount_system_rw(1)
elif choice in ('3'):
addinitd_support(1)
elif choice in ('4'):
brand_boot(1)
elif choice in ('5'):
finalise_boot()
elif choice in ('6'):
custom_remove('working/boot/')
elif choice in ('7'):
custom_deploy('working/boot/')
elif choice in ('8'):
path = 'working/boot/init.rc'
query_add_by_file(path,os.path.join(KitchenConfig.KitchenConfig.KitchenPath
,'processcontrol/queryaddinitrc'))
os.system('sudo ' + KitchenConfig.KitchenConfig.editor + ' ' + path)
elif choice in ('9'):
path = 'working/boot/init.rk30board.rc'
query_add_by_file(path,os.path.join(KitchenConfig.KitchenConfig.KitchenPath
,'processcontrol/queryaddrk30board'))
logging.debug('boot::bootmenu sudo ' + KitchenConfig.KitchenConfig.editor + ' ' + path)
os.system('sudo ' + KitchenConfig.KitchenConfig.editor + ' ' + path)
elif choice in ('W','w'):
browse_boot()
else:
pass
if choice not in ('m','M'):
bootmenu()
except Exception as e:
logerror('boot::bootmenu ',e,1)
