def start_page(self):
req = webdriver.Chrome()
req.get(self.site_url)
time.sleep(3)
html1 = req.page_source
soup = BeautifulSoup(html1, 'lxml')
iframes = soup.findAll('iframe')
self.gs_runner(urldecode(html1))
if iframes:
print('有隐藏页面:', iframes)
for ifr_url in iframes:
if 'https:' not in ifr_url or 'http' not in ifr_url:
ifr_url = urljoin(self.site_url, ifr_url.get('src'))
print('进入隐藏页面：', ifr_url, '，并开启隐藏页面抓取img、css、js的程序........')
try:
html2 = requests.get(ifr_url).text
self.gs_runner(urldecode(html2))
except Exception as e:
print('隐藏页面的地址格式不正确,无法访问：', ifr_url)
# 程序运行完毕，自动关闭浏览器
req.quit()
def test_search(self):
searchkey = "_secr3t.p@ssw()rd's\\te5t!"
self.login_as(self.data.unorm.username, self.data.normpass)
self.selenium.get('%s%s' % (self.live_server_url, reverse('cred.views.list')))
self.waitforload()
searchbox = self.selenium.find_element_by_id("search-box")
searchbox.send_keys(searchkey)
searchbox.send_keys(Keys.ENTER)
self.waitforload()
cur_url = urldecode(self.selenium.current_url)
plan_url = urldecode('%s%s' % (self.live_server_url, reverse('cred.views.list', args=('search', searchkey))))
self.assertEquals(cur_url, plan_url)
def test_search(self):
searchkey = "_secr3t.p@ssw()rd'ste5t!" # FIXME \\ changed to /
self.login_as(self.data.unorm.username, self.data.normpass)
self.selenium.get('%s%s' %
(self.live_server_url, reverse('cred:list')))
self.waitforload()
searchbox = self.selenium.find_element_by_id("search-box")
searchbox.send_keys(searchkey)
searchbox.send_keys(Keys.ENTER)
self.waitforload()
cur_url = urldecode(self.selenium.current_url)
plan_url = urldecode(
'%s%s' % (self.live_server_url,
reverse('cred:list', args=('search', searchkey))))
self.assertEquals(cur_url, plan_url)
def index(self, term=None):
#return '{"customers": [{"id":"123", "name": "hi!"}]}'
r = redis.StrictRedis(host='localhost', port=6379, db=REDIS_DB)
## context data
customers_sorted = []
if term:
customers_result = []
keys = r.keys('*')
if len(keys) > 0:
vals = r.mget(keys)
else:
vals = []
for v in vals:
vobj = json.loads(v)
pattern = '[A-Za-z0-9]*%s' % urldecode(term)
if re.search(pattern, vobj['customer_name'], re.IGNORECASE):
obj = {
'id': vobj['id'],
'name': vobj['customer_name'],
}
# correcting upstream defect with multiple records
# we are cycling through the list of sites to get customers
# producting redundant hits.
if obj not in customers_result:
customers_result.append(obj)
customers_sorted = sorted(customers_result, key=itemgetter('name'))
result = {"customers": customers_sorted}
## render template with context data
return json.dumps(result)
def file_add():
""" Add a file """
file = File(
name=urldecode(request.form['name']),
folder=request.form['folder'],
size=int(request.form['size']),
owner_id=int(current_user.id)
)
db.session.add(file)
db.session.commit()
return app.response_class(response=json.dumps(file.to_object()), mimetype='application/json')
def index(self, term=None):
r = redis.StrictRedis(host='localhost', port=6379, db=REDIS_DB)
engine = tenjin.Engine(path=['views'])
## context data
if term:
customers_result = []
keys = r.keys('*')
if len(keys) > 0:
vals = r.mget(keys)
else:
vals = []
for v in vals:
vobj = json.loads(v)
pattern = '[A-Za-z0-9]*%s' % urldecode(term)
if re.search(pattern, vobj['customer_name'], re.IGNORECASE) or \
re.search(pattern, vobj['site_name'], re.IGNORECASE):
for r in vobj['returns']:
customers_result.append({
'id':
r['id'],
'ts':
r['ts'],
'date':
r['date'],
'customer_name':
vobj['customer_name'],
'site_name':
vobj['site_name'],
'url':
r['url'],
'share_point_url':
r['share_point_url'],
'issue':
r['issue'],
'case_number':
r['case_number']
})
customers_sorted = sorted(customers_result,
key=itemgetter('ts'),
reverse=True)
context = {'keys': customers_sorted}
else:
context = {
'keys': [],
}
## render template with context data
html = engine.render('page.pyhtml', context)
return html
def photo_thumbnail():
""" Add a photo to a gallery """
if current_user.role == "guest":
abort(404)
photo = Photo(name=urldecode(request.form['name']),
ext=request.form['ext'],
gallery_id=request.form['gallery_id'],
aspect_ratio=1.0,
owner_id=1,
created="2000:01:01 00:00:00")
# Tell the thumbnail daemon to generate a thumbnail for this photo
photo.generate_thumbnail()
return app.response_class(response="[]", mimetype='application/json')
def photo_thumbnail():
""" Add a photo to a gallery """
if current_user.role == "guest":
abort(404)
photo = Photo(
name=urldecode(request.form['name']),
ext=request.form['ext'],
gallery_id=request.form['gallery_id'],
aspect_ratio=1.0,
owner_id=1,
created="2000:01:01 00:00:00"
)
# Tell the thumbnail daemon to generate a thumbnail for this photo
photo.generate_thumbnail()
return app.response_class(response="[]", mimetype='application/json')
def decode_bip21_uri(uri):
if not is_bip21_uri(uri):
raise ValueError("Not a valid BIP21 URI: " + uri)
result = {}
parsed = urlparse(uri)
result['address'] = parsed.path
params = parse_qs(parsed.query)
for key in params:
if key.startswith('req-'):
raise ValueError("Unknown required parameter " + key +
" in BIP21 URI.")
if key == 'amount':
amount_str = params['amount'][0]
validate_bip21_amount(amount_str)
# Convert amount to sats, as used internally by JM
result['amount'] = amount_to_sat(amount_str + "btc")
else:
result[key] = urldecode(params[key][0])
return result
def photo_post():
""" Add a photo to a gallery """
if current_user.role == "guest":
abort(404)
photo = Photo(
name=urldecode(request.form['name']),
ext=request.form['ext'],
aspect_ratio=float(request.form['aspect_ratio']),
gallery=Gallery.find_by_id(request.form['gallery_id']),
owner=current_user,
created=request.form['created']
)
# Save the updated photos JSON for this gallery
photo.save()
# Update the gallery modified date
photo.gallery.update_modified()
return app.response_class(response=json.dumps(photo.to_object()), mimetype='application/json')
def photo_post():
""" Add a photo to a gallery """
if current_user.role == "guest":
abort(404)
photo = Photo(name=urldecode(request.form['name']),
ext=request.form['ext'],
aspect_ratio=float(request.form['aspect_ratio']),
gallery_id=request.form['gallery_id'],
owner_id=int(current_user.id),
created=request.form['created'])
# Save the updated photos JSON for this gallery
photo.save()
# Tell the thumbnail daemon to generate a thumbnail for this photo
photo.generate_thumbnail()
# Update the gallery modified date
photo.gallery.updateModified()
return app.response_class(response=json.dumps(photo.to_object()),
mimetype='application/json')
def do_POST(self):
self.do_HEAD()
actions = {}
try:
page, acts = self.path.split("?")
for item in acts.split('&'):
actions[item.split('=')[0]] = item.split('=')[1]
except ValueError:
page = self.path
if (page == '/validate'):
data = urldecode(
self.rfile.read(int(self.headers['content-length'])).decode(
'utf-8')).split('=')[1]
block = myChain.makeBlock(data, float(actions['timestamp']),
int(actions['nonce']))
if myChain.validateBlock(block):
myChain.applyBlock(block)
self.wfile.write("{'applied': 'True'}".encode("utf-8"))
print(" BLOCK APPLIED")
else:
self.wfile.write("{'applied': 'False'}".encode("utf-8"))
print(" BLOCK REJECTED")
def GetEpisodes(url):
page = GetUrl(url)
#Get all the raw links and episode names and put them on a list.
rawLinks = page.xpath('//*[@id="holder-nav"]/li/a/@onclick')
epNames = page.xpath('//*[@id="holder-nav"]/li/a/span/text()')
#check links
epNumber = len(rawLinks)
if epNumber != len(epNames):
print '[!] Error, links missing!'
return False
#Create the episodes list
episodes = []
#Iterate through results and extract links
for n in range(0, epNumber):
cutLinks = rawLinks[n].split("'")
link = cutLinks[1] + urldecode(cutLinks[3])
#Create episode object, set attributes and append to episodes list
episode = Episode(epNames[n], n + 1, link)
episodes.append(episode)
return episodes
def photo_post():
""" Add a photo to a gallery """
if current_user.role == "guest":
abort(404)
photo = Photo(
name=urldecode(request.form['name']),
ext=request.form['ext'],
aspect_ratio=float(request.form['aspect_ratio']),
gallery_id=request.form['gallery_id'],
owner_id=int(current_user.id),
created=request.form['created']
)
# Save the updated photos JSON for this gallery
photo.save()
# Tell the thumbnail daemon to generate a thumbnail for this photo
photo.generate_thumbnail()
# Update the gallery modified date
photo.gallery.updateModified()
return app.response_class(response=json.dumps(photo.to_object()), mimetype='application/json')
def GetEpisodes(url):
page = GetUrl(url)
#Get all the raw links and episode names and put them on a list.
rawLinks = page.xpath('//*[@id="holder-nav"]/li/a/@onclick')
epNames = page.xpath('//*[@id="holder-nav"]/li/a/span/text()')
#check links
epNumber = len(rawLinks)
if epNumber != len(epNames):
print '[!] Error, links missing!'
return False
#Create the episodes list
episodes = []
#Iterate through results and extract links
for n in range(0, epNumber):
cutLinks = rawLinks[n].split("'")
link = cutLinks[1] + urldecode(cutLinks[3])
#Create episode object, set attributes and append to episodes list
episode = Episode(epNames[n], n + 1, link)
episodes.append(episode)
return episodes
def files(request, url):
barre = BarreRaccourcie.objects.all()
documents = Document.objects.filter(folder=urldecode(url)).order_by('file')
folder = urldecode(url)
names = []
types = []
name = 'Folders'
for document in documents:
names.append(document.file.name[8:])
if pathlib.Path(
'media/' +
str(document.file.name)).suffix == '.docx' or pathlib.Path(
'media/' +
str(document.file.name)).suffix == '.odt' or pathlib.Path(
'media/' + str(document.file.name)).suffix == '.doc':
types.append('word')
elif pathlib.Path(
'media/' +
str(document.file.name)).suffix == '.xlx' or pathlib.Path(
'media/' +
str(document.file.name)).suffix == '.xlsx' or pathlib.Path(
'media/' + str(document.file.name)
).suffix == '.csv' or pathlib.Path(
'media/' + str(document.file.name)).suffix == '.CSV':
types.append('excel')
elif pathlib.Path('media/' + str(document.file.name)).suffix == '.apk':
types.append('android')
elif pathlib.Path('media/' + str(document.file.name)).suffix == '.rw3':
types.append('chart-area')
elif pathlib.Path(
'media/' +
str(document.file.name)).suffix == '.pptx' or pathlib.Path(
'media/' + str(document.file.name)).suffix == '.ppt':
types.append('presentation')
elif filetype.guess("media/" + str(document.file.name)):
type = filetype.guess("media/" + str(document.file.name))
types.append(type.mime)
elif pathlib.Path(
'media/' +
str(document.file.name)).suffix == '.py' or pathlib.Path(
'media/' +
str(document.file.name)).suffix == '.js' or pathlib.Path(
'media/' + str(document.file.name)
).suffix == '.c' or pathlib.Path(
'media/' + str(document.file.name)).suffix == '.xml':
types.append('code')
elif pathlib.Path(
'media/' +
str(document.file.name)).suffix == '.avi' or pathlib.Path(
'media/' +
str(document.file.name)).suffix == '.mp4' or pathlib.Path(
'media/' + str(document.file.name)).suffix == '.mkv':
types.append('video')
elif pathlib.Path('media/' +
str(document.file.name)).suffix == '.html':
types.append('html5')
elif pathlib.Path('media/' + str(document.file.name)).suffix == '.css':
types.append('css3')
elif pathlib.Path(
'media/' +
str(document.file.name)).suffix == 'log' or pathlib.Path(
'media/' + str(document.file.name)).suffix == 'config':
types.append('settings')
else:
types.append('file')
return render(
request, 'cloud/particular.html', {
'files': documents,
'url': url,
'names': names,
'types': types,
"barre": barre,
'name': name,
'folder': folder
})
self.base_path = '~/Desktop/gov_spider/' + self.spath
print('base:', self.base_path)
def start_page(self):
req = webdriver.Chrome()
req.get(self.site_url)
time.sleep(3)
html1 = req.page_source
soup = BeautifulSoup(html1, 'lxml')
iframes = soup.findAll('iframe')
self.gs_runner(urldecode(html1))
if iframes:
print('有隐藏页面:', iframes)
for ifr_url in iframes:
if 'https:' not in ifr_url or 'http' not in ifr_url:
ifr_url = urljoin(self.site_url, ifr_url.get('src'))
print('进入隐藏页面：', ifr_url, '，并开启隐藏页面抓取img、css、js的程序........')
try:
html2 = requests.get(ifr_url).text
self.gs_runner(urldecode(html2))
def frac(request, string):
string = urldecode(string)
return render(request, 'TPjs/' + string + '.html')
