def list_members(request, microcosm_id):
offset = int(request.GET.get('offset', 0))
microcosm_url, params, headers = Microcosm.build_request(request.get_host(), id=microcosm_id,
offset=offset, access_token=request.access_token)
request.view_requests.append(grequests.get(microcosm_url, params=params, headers=headers))
try:
responses = response_list_to_dict(grequests.map(request.view_requests))
except APIException as exc:
return respond_with_error(request, exc)
microcosm = Microcosm.from_api_response(responses[microcosm_url])
roles_url, params, headers = RoleList.build_request(request.META['HTTP_HOST'], id=microcosm_id,
offset=offset, access_token=request.access_token)
request.view_requests.append(grequests.get(roles_url, params=params, headers=headers))
try:
responses = response_list_to_dict(grequests.map(request.view_requests))
except APIException as exc:
return respond_with_error(request, exc)
roles = RoleList.from_api_response(responses[roles_url])
view_data = {
'user': Profile(responses[request.whoami_url], summary=False) if request.whoami_url else None,
'site': Site(responses[request.site_url]),
'site_section': 'memberships',
'content': microcosm,
'memberships': roles,
'item_type': 'microcosm',
'pagination': build_pagination_links(responses[roles_url]['roles']['links'], roles.items)
}
return render(request, members_list_template, view_data)
def process_request(self, request):
"""
Checks for access_token cookie and appends it to the request object if present.
All request objects have a view_requests attribute which is a list of requests
that will be executed by grequests to fetch data for the view.
"""
request.access_token = None
request.whoami_url = ''
request.view_requests = []
if request.COOKIES.has_key('access_token'):
# Clean up empty access token.
if request.COOKIES['access_token'] == '':
response = HttpResponseRedirect('/')
response.set_cookie('access_token', '', expires="Thu, 01 Jan 1970 00:00:00 GMT")
return response
request.access_token = request.COOKIES['access_token']
request.whoami_url, params, headers = WhoAmI.build_request(request.get_host(), request.access_token)
request.view_requests.append(grequests.get(request.whoami_url, params=params, headers=headers))
newrelic.agent.add_custom_parameter('access_token', request.access_token[:6])
request.site_url, params, headers = Site.build_request(request.get_host())
request.view_requests.append(grequests.get(request.site_url, params=params, headers=headers))
def geturls(urls,headers=None):
movies = []
movie_hrefs = []
# 使用grequest进行异步获取网页
movies_requests = [grequests.get(u,headers=headers) for u in urls]
movies_datas = grequests.map(movies_requests)
for movies_d in movies_datas:
movies_soup = BeautifulSoup(movies_d.text,'lxml')
names = movies_soup.select('div.item > div.info > div.hd > a > span:nth-of-type(1)')
ranks = movies_soup.select('div.item > div.pic > em')
hrefs = movies_soup.select('div.item > div.info > div.hd > a')
for name,rank,href in zip(names,ranks,hrefs):
movie = MovieClass.Movie(name.get_text(),rank.get_text(),href.get('href'))
movies.append(movie)
for href in hrefs:
movie_hrefs.append(href.get('href'))
# 异步获取250个网页
movie_requests = [grequests.get(u,headers=headers) for u in movie_hrefs]
movie_webs = grequests.map(movie_requests)
# 将网页数据和电影初步数据保存下来
with open('moviehtmls.txt','wb') as fs:
pickle.dump(movie_webs,fs)
with open('moviesdata.txt','wb') as fs:
pickle.dump(movies,fs)
return movies,movie_webs
def handleCommand(self, update):
self.bot.sendChatAction(chat_id=update.message.chat_id, action=telegram.ChatAction.TYPING)
apiServer="localhost"
hacklabRequests = [
grequests.get(url="http://{}/pi_api/temp/".format(apiServer), params={"a":"getTemp"}), # 0: temperature
grequests.get(url="http://{}/pi_api/humidity/".format(apiServer), params={"a":"getHumidity"}), # 1: humidity
grequests.get(url="http://{}/pi_api/gpio/".format(apiServer), params={"a":"readPin", "pin":"1"}), # 2: electronics
grequests.get(url="http://{}/pi_api/gpio/".format(apiServer), params={"a":"readPin", "pin":"0"}), # 3: mechanics
grequests.get(url="http://{}/pi_api/pir/".format(apiServer), params={"a":"getStatus"})] # 4: PIR
hacklabResponses = grequests.map(hacklabRequests)
if None in hacklabResponses:
raise ConnectionError("A request for the /hacklab command failed!")
temperature = json.loads(hacklabResponses[0].text)['data']
humidity = json.loads(hacklabResponses[1].text)['data']
electronicsLight = json.loads(hacklabResponses[2].text)['data'] == "0"
mechanicsLight = json.loads(hacklabResponses[3].text)['data'] == "0"
pirStatus = json.loads(hacklabResponses[4].text)['time']
if not electronicsLight and not mechanicsLight:
lightStatus = "Lights are off. Hacklab is probably empty."
elif electronicsLight != mechanicsLight:
lightStatus = "Lights are on in the {} room.".format(("electronics" if electronicsLight else "mechanics"))
else:
lightStatus = "Lights are on in both rooms!"
responseMessage = "{} Last movement at {}. Temperature is {:.1f}\u00b0C. Humidity is {}%".format(lightStatus, pirStatus, temperature, humidity)
self.bot.sendMessage(chat_id=update.message.chat_id, text=responseMessage)
def server_error(request, exception=None):
view_data = {}
view_requests = []
if request.COOKIES.has_key('access_token'):
request.access_token = request.COOKIES['access_token']
whoami_url, params, headers = WhoAmI.build_request(request.get_host(), request.access_token)
view_requests.append(grequests.get(whoami_url, params=params, headers=headers))
site_url, params, headers = Site.build_request(request.get_host())
view_requests.append(grequests.get(request.site_url, params=params, headers=headers))
responses = response_list_to_dict(grequests.map(view_requests))
if request.whoami_url:
profile = Profile(responses[whoami_url], summary=False)
view_data['user'] = profile
newrelic.agent.add_custom_parameter('profile_name', profile.profile_name)
newrelic.agent.add_custom_parameter('profile_id', profile.id)
newrelic.agent.add_custom_parameter('user_id', profile.user_id)
site = Site(responses[site_url])
view_data['site'] = site
newrelic.agent.add_custom_parameter('site', site.subdomain_key)
# Provide detailed error if returned in the response.
if hasattr(exception, 'detail'):
if exception.detail.has_key('errorDetail'):
view_data['detail'] = exception.detail['errorDetail']
context = RequestContext(request, view_data)
return HttpResponseServerError(loader.get_template('500.html').render(context))
def reap(file_name):
config = ConfigParser.ConfigParser()
config.read('combine.cfg')
inbound_url_file = config.get('Reaper', 'inbound_urls')
outbound_url_file = config.get('Reaper', 'outbound_urls')
with open(inbound_url_file, 'rb') as f:
inbound_urls = [url.rstrip('\n') for url in f.readlines()]
with open(outbound_url_file, 'rb') as f:
outbound_urls = [url.rstrip('\n') for url in f.readlines()]
headers = {'User-Agent': 'harvest.py'}
sys.stderr.write('Fetching inbound URLs\n')
reqs = [grequests.get(url, headers=headers) for url in inbound_urls]
inbound_responses = grequests.map(reqs)
inbound_harvest = [(response.url, response.status_code, response.text) for response in inbound_responses]
sys.stderr.write('Fetching outbound URLs\n')
reqs = [grequests.get(url, headers=headers) for url in outbound_urls]
outbound_responses = grequests.map(reqs)
outbound_harvest = [(response.url, response.status_code, response.text) for response in outbound_responses]
sys.stderr.write('Storing raw feeds in %s\n' % file_name)
harvest = {'inbound': inbound_harvest, 'outbound': outbound_harvest}
with open(file_name, 'wb') as f:
json.dump(harvest, f, indent=2)
def get_pages(response, topics, link_list):
i = 0
nb_link = 0
nb_requests = 0
for res in response:
# get each page from the last
list_pages = []
while int(link_list[i].split('-')[3]) > 0:
previous = link_list[i].split('-')
previous_page = int(previous[3])
previous_page -= 1
previous[3] = str(previous_page)
link_list[i] = '-'.join(previous)
list_pages.append(link_list[i])
print link_list[i]
if nb_requests > 8:
rs = (grequests.get(u) for u in list_pages)
laresponse = grequests.map(rs)
get_pseudos(laresponse)
nb_requests = 0
del list_pages[:]
nb_requests+=1
rs = (grequests.get(u, stream=True) for u in list_pages)
laresponse = grequests.map(rs)
get_pseudos(laresponse)
res.close()
i+=1
return
def parse(url):
page = requests.get(url)
soup = bs(page.text, 'lxml')
if url == start_url:
async_list = []
for i in xrange(1, 6):
print (url+'?&pg={}'.format(i))
rs = (grequests.get(url+'?&pg={}'.format(i), hooks = {'response' : scrape}))
async_list.append(rs)
grequests.map(async_list)
active_sel = soup.find('span', 'zg_selected').find_next()
if active_sel.name == 'ul':
links_list = active_sel.find_all('li')
for link_list in links_list:
link = link_list.find('a')['href'].encode('utf-8')
async_list = []
for i in xrange(1, 6):
print (link+'?&pg={}'.format(i))
rs = (grequests.get(link+'?&pg={}'.format(i), hooks = {'response' : scrape}))
async_list.append(rs)
grequests.map(async_list)
parse(link)
def single(request, profile_id):
"""
Display a single profile by ID.
"""
# Fetch profile details.
profile_url, params, headers = Profile.build_request(request.get_host(), profile_id, access_token=request.access_token)
request.view_requests.append(grequests.get(profile_url, params=params, headers=headers))
# Fetch items created by this profile.
search_q = 'type:conversation type:event type:huddle type:comment authorId:%s' % profile_id
search_params = {'limit': 10, 'q': search_q, 'sort': 'date'}
search_url, params, headers = Search.build_request(request.get_host(), search_params,
access_token=request.access_token)
request.view_requests.append(grequests.get(search_url, params=params, headers=headers))
try:
responses = response_list_to_dict(grequests.map(request.view_requests))
except APIException as exc:
return respond_with_error(request, exc)
user = Profile(responses[request.whoami_url], summary=False) if request.whoami_url else None
profile = Profile(responses[profile_url], summary=False)
view_data = {
'user': user,
'content': profile,
'item_type': 'profile',
'site': Site(responses[request.site_url]),
'search': Search.from_api_response(responses[search_url]),
'site_section': 'people'
}
return render(request, single_template, view_data)
def main():
resource.setrlimit(resource.RLIMIT_NOFILE, (2048, 2048))
hashes = set()
past_urls = set()
args = setup_args(sys.argv[1:])
if args.config:
cfg = config(args, args.config)
else:
cfg = config(args, 'maltrieve.cfg')
cfg.check_proxy()
hashes = load_hashes('hashes.json')
past_urls = load_urls('urls.json')
print "Processing source URLs"
# TODO: Replace with plugins
source_urls = {'https://zeustracker.abuse.ch/monitor.php?urlfeed=binaries': process_xml_list_desc,
'http://www.malwaredomainlist.com/hostslist/mdl.xml': process_xml_list_desc,
'http://malc0de.com/rss/': process_xml_list_desc,
'http://vxvault.net/URL_List.php': process_simple_list,
'http://urlquery.net/': process_urlquery,
'http://support.clean-mx.de/clean-mx/rss?scope=viruses&limit=0%2C64': process_xml_list_title,
'http://malwareurls.joxeankoret.com/normal.txt': process_simple_list}
headers = {'User-Agent': 'Maltrieve'}
reqs = [grequests.get(url, timeout=60, headers=headers, proxies=cfg.proxy) for url in source_urls]
source_lists = grequests.map(reqs)
print "Completed source processing"
headers['User-Agent'] = cfg.useragent
malware_urls = set()
for response in source_lists:
if hasattr(response, 'status_code') and response.status_code == 200:
malware_urls.update(source_urls[response.url](response.text))
if cfg.inputfile:
with open(cfg.inputfile, 'rb') as f:
moar_urls = list(f)
malware_urls.update(moar_urls)
print "Downloading samples, check log for details"
malware_urls -= past_urls
reqs = [grequests.get(url, timeout=60, headers=headers, proxies=cfg.proxy) for url in malware_urls]
for chunk in chunker(reqs, 32):
malware_downloads = grequests.map(chunk)
for each in malware_downloads:
if not each or each.status_code != 200:
continue
if save_malware(each, cfg):
past_urls.add(each.url)
print "Completed downloads"
save_urls(past_urls, 'urls.json')
save_hashes(hashes, 'hashes.json')
sort_downloads()
def forbidden(request):
view_data = {}
view_requests = []
if request.COOKIES.has_key('access_token'):
request.access_token = request.COOKIES['access_token']
whoami_url, params, headers = WhoAmI.build_request(request.get_host(), request.access_token)
view_requests.append(grequests.get(whoami_url, params=params, headers=headers))
site_url, params, headers = Site.build_request(request.get_host())
view_requests.append(grequests.get(request.site_url, params=params, headers=headers))
responses = response_list_to_dict(grequests.map(view_requests))
if request.whoami_url:
profile = Profile(responses[whoami_url], summary=False)
view_data['user'] = profile
newrelic.agent.add_custom_parameter('profile_name', profile.profile_name)
newrelic.agent.add_custom_parameter('profile_id', profile.id)
newrelic.agent.add_custom_parameter('user_id', profile.user_id)
site = Site(responses[site_url])
view_data['site'] = site
newrelic.agent.add_custom_parameter('site', site.subdomain_key)
context = RequestContext(request, view_data)
return HttpResponseForbidden(loader.get_template('403.html').render(context))
def parse(url):
if url in start_urls:
async_list = []
print (url)
for i in xrange(1, 6):
rs = (grequests.get(url+'?&pg={}'.format(i), hooks = {'response' : scrape}))
async_list.append(rs)
grequests.map(async_list)
page = requests.get(url)
soup = bs(page.text, 'lxml')
try:
active_sel = soup.find('span', 'zg_selected').find_next()
if active_sel.name == 'ul':
links_lists= active_sel.find_all('li')
asins = pool.map(multiparse, links_lists)
for asin in asins:
async_list = []
print (asin)
for i in xrange(1, 6):
rs = (grequests.get(asin+'?&pg={}'.format(i), hooks = {'response' : scrape}))
async_list.append(rs)
parse(asin)
grequests.map(async_list)
except:
parse(url)
def parse(url):
if url in start_urls:
async_list = []
print(url)
rs = grequests.get(url, headers=headers, hooks={"response": scrape})
async_list.append(rs)
grequests.map(async_list)
page = requests.get(url)
soup = bs(page.text, "lxml")
try:
active_sel = soup.find("span", "zg_selected").find_next()
if active_sel.name == "ul":
links_lists = active_sel.find_all("li")
asins = pool.map(multiparse, links_lists)
for asin in asins:
async_list = []
print(asin)
for i in xrange(1, 6):
rs = grequests.get(asin + "?&pg={}".format(i), hooks={"response": scrape})
async_list.append(rs)
parse(asin)
grequests.map(async_list)
except:
parse(url)
def get_route_info_from_infotraffic(known_lines_csv: str, known_stations_csv: str)-> Dict[int, Tuple[Route, Route]]:
root = 'http://86.122.170.105:61978/html/timpi/'
urls = [grequests.get(root + 'tram.php', stream=False),
grequests.get(root + 'trol.php', stream=False),
grequests.get(root + 'auto.php', stream=False)]
known_lines = { line.line_id: line for line in importer.parse_lines_from_csv(known_lines_csv) }
known_lines = known_lines # type: Dict[int, Line]
known_stations = { station.raw_name: station for station in importer.parse_stations_from_csv(known_stations_csv) }
known_stations = known_stations # type: Dict[str, Station]
line_id_re = re.compile("param1=(\d+)")
line_id_to_routes = {} # type: Dict[int, Tuple[Route, Route]]
for page in grequests.imap(urls, size=len(urls), exception_handler=exception_handler):
page.raise_for_status()
if page.status_code == requests.codes.ok:
soup = bs4.BeautifulSoup(page.text, "html.parser")
unknown_lines = { } # type: Dict[int, str]
line_requests = []
for a in soup.select("div p a"):
line_id = int(line_id_re.search(a['href']).group(1))
line = known_lines.get(line_id, None)
if not line:
line_name = a['title'] if a.has_attr('title') else None
if line_name is None:
img = a.select("img")[0]
line_name = img['alt'] if img and img.has_attr('alt') else 'unknown'
unknown_lines[line_id] = line_name
print("WARNING: unknown line '{line_name}' (line ID: {line_id}) encountered at {url}"
.format(line_name=line_name, line_id=line_id, url=page.url))
line_requests.append(grequests.get(root + a['href'], stream=False))
for line_response in grequests.imap(line_requests, size=6, exception_handler=exception_handler):
line_id = int(line_id_re.search(line_response.url).group(1))
routes = parse_arrivals_from_infotrafic(line_id, known_stations, line_response, include_unknown_stations=True)
line = known_lines.get(line_id, None)
line_name = line.line_name if line is not None else unknown_lines.get(line_id, "unknown")
route1 = route2 = None
for route_id, route in enumerate(routes):
valid_stations = []
for station, arrival in route:
if not isinstance(station, Station):
print("WARNING: unknown station '{raw_station_name}' encountered in route {route_id} of line {line_name} (line ID: {line_id})"
.format(line_name=line_name, line_id=line_id, route_id=route_id, raw_station_name=station))
else:
if not station.lng or not station.lat:
print("WARNING: station '{station_name}' (station ID: {station_id}) has no GPS coordinates defined"
.format(station_name=station.friendly_name, station_id=station.station_id))
valid_stations.append(station)
if valid_stations and line is not None:
if route_id == 0:
route1 = Route(route_id, line.route_name_1, line.line_id, valid_stations)
elif route_id == 1:
route2 = Route(route_id, line.route_name_2, line.line_id, valid_stations)
if route1 is not None and route2 is not None:
line_id_to_routes[line.line_id] = (route1, route2)
return line_id_to_routes
def test_get_connection_managers_info(
api_backend,
api_test_context,
api_raiden_service):
# check that no connection managers exists yet
request = grequests.get(
api_url_for(api_backend, 'connectionmanagersresource')
)
response = request.send().response
token_addresses = response.json()
assert token_addresses == dict()
funds = 100
token_address1 = '0xea674fdde714fd979de3edf0f56aa9716b898ec8'
connect_data_obj = {
'funds': funds,
}
request = grequests.put(
api_url_for(api_backend, 'connectionsresource', token_address=token_address1),
json=connect_data_obj,
)
response = request.send().response
assert_no_content_response(response)
# check that there now is one registered channel manager
request = grequests.get(
api_url_for(api_backend, 'connectionmanagersresource')
)
response = request.send().response
token_addresses = response.json()
assert isinstance(token_addresses, dict) and len(token_addresses.keys()) == 1
assert token_address1 in token_addresses
assert isinstance(token_addresses[token_address1], dict)
assert set(token_addresses[token_address1].keys()) == {'funds', 'sum_deposits', 'channels'}
funds = 100
token_address2 = '0x3edf0f56aa9716b898ec8ea674fdde714fd979de'
connect_data_obj = {
'funds': funds,
}
request = grequests.put(
api_url_for(api_backend, 'connectionsresource', token_address=token_address2),
json=connect_data_obj,
)
response = request.send().response
assert_no_content_response(response)
# check that there now are two registered channel managers
request = grequests.get(
api_url_for(api_backend, 'connectionmanagersresource')
)
response = request.send().response
token_addresses = response.json()
assert isinstance(token_addresses, dict) and len(token_addresses.keys()) == 2
assert token_address2 in token_addresses
assert isinstance(token_addresses[token_address2], dict)
assert set(token_addresses[token_address2].keys()) == {'funds', 'sum_deposits', 'channels'}
def test_break_blockchain_events(
api_backend,
api_test_context,
api_raiden_service,
rest_api_port_number):
api_test_context.add_events([{
'_event_type': 'ChannelNew',
'settle_timeout': 10,
'netting_channel': '0xea674fdde714fd979de3edf0f56aa9716b898ec8',
'participant1': '0x4894a542053248e0c504e3def2048c08f73e1ca6',
'participant2': '0x356857Cd22CBEFccDa4e96AF13b408623473237A',
'block_number': 15,
}, {
'_event_type': 'ChannelSettled',
'block_number': 35,
}])
# Let's make sure that token_address as a query argument does not override
# the provided token_address
api_test_context.specify_token_for_channelnew('0x61c808d82a3ac53231750dadc13c777b59310bd9')
request = grequests.get(
'http://localhost:{port}/api/1/events/tokens/0x61c808d82a3ac53231750dadc13c777b59310bd9'
'?from_block=5&to_block=20'
'&token_address=0x167a9333bf582556f35bd4d16a7e80e191aa6476'.format(
port=rest_api_port_number,
)
)
response = request.send().response
assert_proper_response(response)
response = json.loads(response.text)
assert len(response) == 1
assert response[0] == {
'event_type': 'ChannelNew',
'settle_timeout': 10,
'netting_channel': '0xea674fdde714fd979de3edf0f56aa9716b898ec8',
'participant1': '0x4894a542053248e0c504e3def2048c08f73e1ca6',
'participant2': '0x356857Cd22CBEFccDa4e96AF13b408623473237A',
'block_number': 15,
}
# Assert the same for the event/channels endpoint
api_test_context.specify_channel_for_events('0xedbaf3c5100302dcdda53269322f3730b1f0416d')
request = grequests.get(
'http://localhost:{port}/api/1/events/channels/0xedbaf3c5100302dcdda53269322f3730b1f0416d'
'?from_block=10&to_block=90'
'&channel_address=0x167A9333BF582556f35Bd4d16A7E80E191aa6476'.format(
port=rest_api_port_number,
)
)
response = request.send().response
assert_proper_response(response)
response = json.loads(response.text)
assert len(response) == 1
assert response[0] == {
'event_type': 'ChannelSettled',
'block_number': 35,
}
def test_map_timeout_no_exception_handler(self):
"""
compliance with existing 0.2.0 behaviour
"""
reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))]
responses = grequests.map(reqs)
self.assertIsNone(responses[0])
self.assertTrue(responses[1].ok)
self.assertEqual(len(responses), 2)
def reap(file_name):
config = ConfigParser.SafeConfigParser(allow_no_value=False)
cfg_success = config.read('combine.cfg')
if not cfg_success:
logger.error('Reaper: Could not read combine.cfg.')
logger.error('HINT: edit combine-example.cfg and save as combine.cfg.')
return
inbound_url_file = config.get('Reaper', 'inbound_urls')
outbound_url_file = config.get('Reaper', 'outbound_urls')
try:
with open(inbound_url_file, 'rb') as f:
inbound_urls = [url.rstrip('\n') for url in f.readlines()]
except EnvironmentError as e:
logger.error('Reaper: Error while opening "%s" - %s' % (inbound_url_file, e.strerror))
return
try:
with open(outbound_url_file, 'rb') as f:
outbound_urls = [url.rstrip('\n') for url in f.readlines()]
except EnvironmentError as e:
logger.error('Reaper: Error while opening "%s" - %s' % (outbound_url_file, e.strerror))
return
logger.info('Fetching inbound URLs')
inbound_files=[]
for url in inbound_urls:
if url.startswith('file://'):
inbound_files.append(url.partition('://')[2])
inbound_urls.remove(url)
headers = {'User-Agent': 'harvest.py'}
reqs = [grequests.get(url, headers=headers) for url in inbound_urls]
inbound_responses = grequests.map(reqs)
inbound_harvest = [(response.url, response.status_code, response.text) for response in inbound_responses if response]
for each in inbound_files:
with open(each,'rb') as f:
inbound_harvest.append(('file://'+each, 200, f.read()))
logger.info('Fetching outbound URLs')
outbound_files=[]
for url in outbound_urls:
if url.startswith('file://'):
outbound_files.append(url.partition('://')[2])
outbound_urls.remove(url)
reqs = [grequests.get(url, headers=headers) for url in outbound_urls]
outbound_responses = grequests.map(reqs)
outbound_harvest = [(response.url, response.status_code, response.text) for response in outbound_responses if response]
for each in outbound_files:
with open(each,'rb') as f:
outbound_harvest.append(('file://'+each, 200, f.read()))
logger.error('Storing raw feeds in %s' % file_name)
harvest = {'inbound': inbound_harvest, 'outbound': outbound_harvest}
with open(file_name, 'wb') as f:
json.dump(harvest, f, indent=2)
def test_map_timeout_exception_handler_returns_false(self):
"""
if you don't want your exceptions to show up in the map result
"""
def exception_handler(request, exception):
return False
reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))]
responses = grequests.map(reqs, exception_handler=exception_handler)
self.assertTrue(responses[0].ok)
self.assertEqual(len(responses), 1)
def test_map_timeout_exception_handler_returns_exception(self):
"""
ensure returned value from exception handler is stuffed in the map result
"""
def exception_handler(request, exception):
return exception
reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))]
responses = grequests.map(reqs, exception_handler=exception_handler)
self.assertIsInstance(responses[0], Timeout)
self.assertTrue(responses[1].ok)
self.assertEqual(len(responses), 2)
def test_map_timeout_exception_handler_no_return(self):
"""
ensure default behaviour for a handler that returns None
"""
def exception_handler(request, exception):
pass
reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))]
responses = grequests.map(reqs, exception_handler=exception_handler)
self.assertIsNone(responses[0])
self.assertTrue(responses[1].ok)
self.assertEqual(len(responses), 2)
def main(self, server, port):
logging.basicConfig(filename='logs/reader.log', level=logging.DEBUG)
logging.info("Attempting to set up the reader on: %s:%d" % (server,
port))
try:
"""
ls /dev/serial/by-id/ to get all connected devices.
Use that to determine this
/dev/serial/by-id/usb-FTDI_FT232R_USB_UART_AD026CI5-if00-port0
"""
usb_devices = glob.glob("/dev/serial/by-id/*")
ser = serial.Serial(usb_devices[0], 9600, timeout=0)
logging.info("Crosstag reader is up and running.")
sys.stdout.flush()
except:
now = datetime.now()
logging.error('%s reader ERROR unable to setup the RFID reade,",\
" is it plugged in?' % (now))
sys.exit(-1)
while True:
if ser.inWaiting() > 0:
data = ser.readline()
if len(data) < 12:
continue
now = datetime.now()
ser.flushOutput()
data = data.strip()
tag_nbr = data[1:]
if len(tag_nbr) != 12:
logging.error('%s reader ERROR [%s] is too long.",\
" len(tag_nbr): %d' % (now, tag_nbr,
len(tag_nbr)))
# TODO, PiFace code to ask the person to tag again.
# audio signal and visual signal.
continue
try:
print('%s reader tagging [%s]' % (now, tag_nbr))
urls = ["http://%s:%d/crosstag/v1.0/tagevent/%s" % (server, port, tag_nbr)]
unsent = (grequests.get(url) for url in urls)
res = grequests.map(unsent)
res = grequests.get("http://%s:%d/crosstag/v1.0/tagevent/%s" % (server, port, tag_nbr), timeout=3)
now = datetime.now()
logging.info("%s reader tagging result: [%s]" % (now, tag_nbr))
except: # catch *all* exceptions
e = sys.exc_info()[0]
logging.error('%s reader failed (timeout?) to tag %s error: %s' % (now, tag_nbr, e))
else:
time.sleep(1)
def settings(request):
if request.method == 'POST':
# Update global settings for notifications.
postdata = {
'sendEmail': bool(request.POST.get('profile_receive_email')),
'sendSMS': False,
}
try:
GlobalOptions.update(request.get_host(), postdata, request.access_token)
except APIException as exc:
return respond_with_error(request, exc)
# Update settings for each notification type.
for x in range(1, 10):
if request.POST.get('id_' + str(x)):
postdata = {
'id': int(request.POST['id_' + str(x)]),
'sendEmail': bool(request.POST.get('send_email_' + str(x))),
'sendSMS': False,
}
try:
UpdatePreference.update(request.get_host(), request.POST['id_' + str(x)],
postdata, request.access_token)
except APIException as exc:
return respond_with_error(request, exc)
return HttpResponseRedirect(reverse('updates-settings'))
if request.method == 'GET':
url, params, headers = UpdatePreference.build_request(request.get_host(), request.access_token)
request.view_requests.append(grequests.get(url, params=params, headers=headers))
url2, params2, headers2 = GlobalOptions.build_request(request.get_host(), request.access_token)
request.view_requests.append(grequests.get(url2, params=params2, headers=headers2))
try:
responses = response_list_to_dict(grequests.map(request.view_requests))
except APIException as exc:
return respond_with_error(request, exc)
preference_list = UpdatePreference.from_list(responses[url])
global_options = GlobalOptions.from_api_response(responses[url2])
view_data = {
'user': Profile(responses[request.whoami_url], summary=False),
'site': Site(responses[request.site_url]),
'content': preference_list,
'globaloptions': global_options,
}
return render(request, settings_template, view_data)
def confirm(request):
"""
View for moderation actions on a single item.
"""
if request.method == 'GET':
if request.GET.get('item_type') == 'conversation':
url, params, headers = Conversation.build_request(
request.get_host(),
request.GET.get('item_id'),
access_token=request.access_token
)
request.view_requests.append(grequests.get(url, params=params, headers=headers))
responses = response_list_to_dict(grequests.map(request.view_requests))
content = Conversation.from_api_response(responses[url])
elif request.GET.get('item_type') == 'event':
url, params, headers = Event.build_request(
request.get_host(),
request.GET.get('item_id'),
access_token=request.access_token
)
request.view_requests.append(grequests.get(url, params=params, headers=headers))
responses = response_list_to_dict(grequests.map(request.view_requests))
content = Event.from_api_response(responses[url])
elif request.GET.get('item_type') == 'microcosm':
url, params, headers = Microcosm.build_request(
request.get_host(),
request.GET.get('item_id'),
access_token=request.access_token
)
request.view_requests.append(grequests.get(url, params=params, headers=headers))
responses = response_list_to_dict(grequests.map(request.view_requests))
content = Microcosm.from_api_response(responses[url])
view_data = {
'user': Profile(responses[request.whoami_url], summary=False),
'site': Site(responses[request.site_url]),
'content': content,
'item_type': request.GET.get('item_type'),
'action': request.GET.get('action'),
}
if request.GET.get('action') == 'move':
# Fetch list of microcosms to supply in form.
view_data['microcosms'] = MicrocosmList.retrieve(
request.get_host(),
access_token=request.access_token
)
return render(request, 'forms/moderation_item.html', view_data)
def edit_members(request, microcosm_id, group_id):
if request.method == 'POST':
pass
elif request.method == 'GET':
try:
offset = int(request.GET.get('offset', 0))
except ValueError:
offset = 0
microcosm_url, params, headers = Microcosm.build_request(request.get_host(), id=microcosm_id,
offset=offset, access_token=request.access_token)
request.view_requests.append(grequests.get(microcosm_url, params=params, headers=headers))
role_url, params, headers = Role.build_request(request.get_host(), microcosm_id=microcosm_id,
id=group_id, offset=offset, access_token=request.access_token)
request.view_requests.append(grequests.get(role_url, params=params, headers=headers))
criteria_url, params, headers = RoleCriteriaList.build_request(request.get_host(),
microcosm_id=microcosm_id, id=group_id, offset=offset, access_token=request.access_token)
request.view_requests.append(grequests.get(criteria_url, params=params, headers=headers))
profiles_url, params, headers = RoleProfileList.build_request(request.get_host(),
microcosm_id=microcosm_id, id=group_id, offset=offset, access_token=request.access_token)
request.view_requests.append(grequests.get(profiles_url, params=params, headers=headers))
try:
responses = response_list_to_dict(grequests.map(request.view_requests))
except APIException as exc:
return respond_with_error(request, exc)
microcosm = Microcosm.from_api_response(responses[microcosm_url])
role = Role.from_api_response(responses[role_url])
criteria = RoleCriteriaList(responses[criteria_url])
profiles = RoleProfileList(responses[profiles_url])
view_data = {
'user': Profile(responses[request.whoami_url], summary=False) if request.whoami_url else None,
'site': Site(responses[request.site_url]),
'site_section': 'memberships',
'content': microcosm,
'role': role,
'criteria': criteria,
'profiles': profiles,
'item_type': 'memberships',
'state_edit': True,
'pagination': build_pagination_links(responses[microcosm_url]['items']['links'], microcosm.items)
}
return render(request, members_form_template, view_data)
def test_api_get_channel_list(
api_backend,
token_addresses,
reveal_timeout,
):
partner_address = '0x61C808D82A3Ac53231750daDc13c777b59310bD9'
request = grequests.get(
api_url_for(
api_backend,
'channelsresource',
),
)
response = request.send().response
assert_proper_response(response, HTTPStatus.OK)
assert response.json() == []
# let's create a new channel
token_address = token_addresses[0]
settle_timeout = 1650
channel_data_obj = {
'partner_address': partner_address,
'token_address': to_checksum_address(token_address),
'settle_timeout': settle_timeout,
'reveal_timeout': reveal_timeout,
}
request = grequests.put(
api_url_for(
api_backend,
'channelsresource',
),
json=channel_data_obj,
)
response = request.send().response
assert_proper_response(response, HTTPStatus.CREATED)
request = grequests.get(
api_url_for(
api_backend,
'channelsresource',
),
)
response = request.send().response
assert_proper_response(response, HTTPStatus.OK)
channel_info = response.json()[0]
assert channel_info['partner_address'] == partner_address
assert channel_info['token_address'] == to_checksum_address(token_address)
assert 'token_network_identifier' in channel_info
def bulkRequests(topic_list): start_time = time.time() debugFunction() try: rs = (grequests.get(u, timeout=5) for u in topic_list) response = grequests.map(rs) return response except: s = (grequests.get(u, timeout=5) for u in topic_list) response = grequests.map(rs) return response elapsedTime(start_time) return
def get_information_from_page(url_list,asynchronous=False):
if asynchronous:
for urls in url_list:
rs = (grequests.get(u,stream=False) for u in urls)
responses = grequests.map(rs)
results = []
for r in responses:
result = {}
html = lxml.html.fromstring(r.text)
posting_body = html.xpath('//div[@class="postingBody"]')
result["textbody"] = [i.text_content() for i in posting_body]
result['pictures'] = html.xpath('//ul[@id="viewAdPhotoLayout"]/li/a/@href')
result['url'] = r.url
results.append(result)
r.close()
return results
else:
r = requests.get(url_list)
html = lxml.html.fromstring(r.text)
posting_body = html.xpath('//div[@class="postingBody"]')
textbody = [i.text_content() for i in posting_body]
pictures = html.xpath('//ul[@id="viewAdPhotoLayout"]/li/a/@href')
return textbody,pictures
def list(request):
# Offset for paging of huddles
try:
offset = int(request.GET.get('offset', 0))
except ValueError:
offset = 0
huddle_url, params, headers = HuddleList.build_request(request.get_host(), offset=offset,
access_token=request.access_token)
request.view_requests.append(grequests.get(huddle_url, params=params, headers=headers))
try:
responses = response_list_to_dict(grequests.map(request.view_requests))
except APIException as exc:
return respond_with_error(request, exc)
huddles = HuddleList(responses[huddle_url])
view_data = {
'user': Profile(responses[request.whoami_url], summary=False) if request.whoami_url else None,
'site': Site(responses[request.site_url]),
'content': huddles,
'pagination': build_pagination_links(responses[huddle_url]['huddles']['links'], huddles.huddles)
}
return render(request, list_template, view_data)
def process_request(self, request):
"""
Checks for access_token cookie and appends it to the request object if present.
All request objects have a view_requests attribute which is a list of requests
that will be executed by grequests to fetch data for the view.
"""
request.access_token = None
request.whoami_url = ''
request.view_requests = []
request.site = None
if request.COOKIES.has_key('access_token'):
request.access_token = request.COOKIES['access_token']
url, params, headers = WhoAmI.build_request(request.get_host(), request.access_token)
request.view_requests.append(grequests.get(url, params=params, headers=headers))
request.whoami_url = url
try:
request.site = Site.retrieve(request.get_host())
except APIException as e:
if e.status_code == 400:
return HttpResponseRedirect('https://microco.sm')
return None
def 模具一一gr无序网址列表请求返回网页内容(self, 网址列表, 分流数=100,并发数=20): # grequests.imap(任务列
初始网址列表数 = len(网址列表)
条件循环 = 1
返回网页一链接组列表 = []
while 条件循环 == 1:
此时数 = int(time.time())
if 此时数 > self.换IP时间计数 + 60:
self.模具一一高位换头部信息()
self.模具一一换ip连接()
网址列表数 = len(网址列表)
任务列表 = []
网址分列表 = []
for 各帖子链接 in 网址列表[0:分流数]:
网址分列表.append(各帖子链接)
if len(网址列表) > 分流数 - 1: # break # 结束循环 continue # 跳过当前循环,继续进行下一轮循环
网址列表 = 网址列表[分流数:]
else: # 否则
网址列表 = []
for 各帖子链接 in 网址分列表:
任务 = grequests.get(各帖子链接, headers=self.头部信息) # timeout=len(任务列表)//2,
任务列表.append(任务)
try: # 调用异常处理,应对易发生错误的位置
返回网页内容集 = grequests.imap(任务列表, size=并发数) # size=3 并发数 3 gtimeout超时时间
except (requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout,
requests.exceptions.ConnectionError, requests.exceptions.ConnectTimeout,
requests.exceptions.ChunkedEncodingError, requests.exceptions.InvalidSchema) as 异常:
print('gr无序异常，倒数6秒再连接', 异常)
# 网址列表.append(返回网页内容.url)
time.sleep(3)
else:
for 返回网页内容 in 返回网页内容集:
网页内容文本 = str(返回网页内容)
if '<Response [200]>' in 网页内容文本 and "访问过于频繁" not in 返回网页内容.text:
# print('返回网页内容', 返回网页内容)
返回网页内容.encoding = "UTF-8"
返回网页一链接组 = []
返回网页一链接组.append(返回网页内容)
返回网页一链接组.append(返回网页内容.url)
返回网页一链接组列表.append(返回网页一链接组)
print('成功采集网页：', 返回网页内容.url)
else: # 否则
网址列表.append(返回网页内容.url)
此时数 = int(time.time())
if len(网址列表) == 0:
print('网页采集,全部完成')
条件循环 = 998
elif 此时数 > self.换IP时间计数 + 初始网址列表数 * 3:
print('网页未完成采集数:', 初始网址列表数 - len(返回网页一链接组列表))
条件循环 = 998
#time.sleep(2) # 等待
return 返回网页一链接组列表 # 返回
This works faster than the solution in the book by making asynchronous
requests with grequests (https://github.com/spyoungtech/grequests)
'''
import grequests
from urllib.request import urlopen
from bs4 import BeautifulSoup
#get the url
page_url = input("Which web page do you want to analyze? ")
try:
page = urlopen(page_url)
except:
print("Cannot open %s" % page_url)
quit()
#get all the hlinks
soup = BeautifulSoup(page, features="lxml")
links = [(link["href"]) for link in soup.find_all("a") if link.has_attr("href")]
# This will notify about and print broken links
def exception_handler(request, exception):
print("This link might be broken:")
print(exception)
# Create a set of unsent Requests:
rs = (grequests.get(u) for u in links)
# Send them all at the same time:
grequests.map(rs, exception_handler=exception_handler)
def query_get_task_with_details(bot_memo, present_skill, bot_nlp):
if ((bot_memo == {} or bot_memo['index']) and present_skill == 'get_task'):
#requests can be used for synchronous requests
# r = requests.get("https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json", auth=HTTPBasicAuth('pritamsa', 'rupu@0801'))
# body1 = r.json()
#grequests is faster
url1 = [
"https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json"
]
rs1 = (grequests.get(u, auth=('pritamsa', 'rupu@0801')) for u in url1)
#both imap and map can be used
#reque = grequests.imap(rs,size=1)
reque1 = grequests.map(rs1, size=1)
response_array1 = []
for response1 in reque1:
print(response1)
x1 = response1.json()
response_array1.append(x1)
body1 = response_array1[0]
no_of_tasks = len(body1["d"]["results"])
if (body1["d"]["results"]):
#task details
instance_id = body1["d"]["results"][0]["InstanceID"]
task_title = body1["d"]["results"][0]["TaskTitle"]
scrapped_po_no = task_title.split("order ", 1)[1]
body2, body3 = take_action_async(scrapped_po_no)
#po_header detail
created_by_user = body2["d"]["CreatedByUser"]
SupplierName = body2["d"]["SupplierName"]
PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]
DocumentCurrency = body2["d"]["DocumentCurrency"]
PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]
final_reply_string = ''
concat_string_for_multiple_lineitems = ''
per_item_desc_dict = {}
all_item_details = {}
#po item detail
no_of_line_items = len(body3["d"]["results"])
for i in range(no_of_line_items):
Material = body3["d"]["results"][i]["Material_Text"]
Plant = body3["d"]["results"][i]["Plant"]
OrderQuantity = body3["d"]["results"][i]["OrderQuantity"]
netPriceItem = body3["d"]["results"][i]["NetPriceAmount"]
documentCurrency = body3["d"]["results"][i]["DocumentCurrency"]
price_present_item_with_currency = netPriceItem + documentCurrency
item_no = 'item : ' + str(i + 1)
# print(item_no)
#item_no = dict(item_no)
per_item_desc_dict = {
item_no: {
'Material': Material,
'Plant': Plant,
'OrderQuantity': OrderQuantity,
'netPriceItem': price_present_item_with_currency
}
}
all_item_details.update(per_item_desc_dict)
#use this when sending the item details as string all in one reply
# concat_string_for_multiple_lineitems = concat_string_for_multiple_lineitems \
# + 'Material: ' + Material + '.\n' + 'plant: ' + Plant + '.\n' \
# + 'OrderQuantity: ' + OrderQuantity + '.\n'
get_task_string = ''
get_task_string_with_header_detail = ''
get_task_string = task_title + '.' + '\n'
get_task_string_with_header_detail = 'created by user: '******'.' + '\n' + 'SupplierName: ' + SupplierName \
+ '.' + '\n' + 'PurchaseOrderNetAmount: ' + PurchaseOrderNetAmount + ' ' + DocumentCurrency + '.'+'\n'
#final_reply_string = 'Now you have got, '+ str(no_of_tasks) + ' pending tasks to approve. ' + get_task_string + get_task_string_with_header_detail +'You have: ' + str(no_of_line_items) +' items.\n'+ concat_string_for_multiple_lineitems + " say approve to approve this task or say ignore to skip this task and move on to your next task, or say next to get your next task with details."
final_reply_string = 'Now you have got, ' + str(
no_of_tasks
) + ' pending tasks to approve. ' + get_task_string + get_task_string_with_header_detail + 'You have: ' + str(
no_of_line_items
) + ' items.\n' + " say get item details to get all the item details in this purchase order. Or,say approve to approve this task or say ignore to skip this task and move on to your next task, or say next to get your next task with details."
return final_reply_string, 1, instance_id, created_by_user, SupplierName, (
PurchaseOrderNetAmount + ' ' + DocumentCurrency
), '', all_item_details, no_of_line_items, scrapped_po_no #return 1for memory index as no memo is present in the beggining
else:
final_reply_string = 'no more tasks to approve in your inbox.'
return final_reply_string, 1, bot_memo, bot_memo, bot_memo, bot_memo, '', '', '', bot_memo
elif (
(bot_memo['index']) and
(present_skill == 'get_next_task' or present_skill == 'ignore_task')):
#requests can be used for synchronous requests
# r = requests.get("https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json", auth=HTTPBasicAuth('pritamsa', 'rupu@0801'))
# body1 = r.json()
#grequests is faster
url1 = [
"https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json"
]
rs1 = (grequests.get(u, auth=('pritamsa', 'rupu@0801')) for u in url1)
#both imap and map can be used
#reque = grequests.imap(rs,size=1)
reque1 = grequests.map(rs1, size=1)
response_array1 = []
for response1 in reque1:
print(response1)
x1 = response1.json()
response_array1.append(x1)
body1 = response_array1[0]
no_of_tasks = len(body1["d"]["results"])
if ((len(body1["d"]["results"]) == 1)):
instance_id = body1["d"]["results"][0]["InstanceID"]
task_title = body1["d"]["results"][0]["TaskTitle"]
scrapped_po_no = task_title.split("order ", 1)[1]
body2, body3 = take_action_async(scrapped_po_no)
#po_header detail
created_by_user = body2["d"]["CreatedByUser"]
SupplierName = body2["d"]["SupplierName"]
PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]
DocumentCurrency = body2["d"]["DocumentCurrency"]
PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]
final_reply_string = ''
concat_string_for_multiple_lineitems = ''
per_item_desc_dict = {}
all_item_details = {}
#po item detail
no_of_line_items = len(body3["d"]["results"])
for i in range(no_of_line_items):
Material = body3["d"]["results"][i]["Material_Text"]
Plant = body3["d"]["results"][i]["Plant"]
OrderQuantity = body3["d"]["results"][i]["OrderQuantity"]
netPriceItem = body3["d"]["results"][i]["NetPriceAmount"]
documentCurrency = body3["d"]["results"][i]["DocumentCurrency"]
price_present_item_with_currency = netPriceItem + documentCurrency
item_no = 'item : ' + str(i + 1)
# print(item_no)
#item_no = dict(item_no)
per_item_desc_dict = {
item_no: {
'Material': Material,
'Plant': Plant,
'OrderQuantity': OrderQuantity,
'netPriceItem': price_present_item_with_currency
}
}
all_item_details.update(per_item_desc_dict)
#use this when sending the item details as string all in one reply
# concat_string_for_multiple_lineitems = concat_string_for_multiple_lineitems \
# + 'Material: ' + Material + '.\n' + 'plant: ' + Plant + '.\n' \
# + 'OrderQuantity: ' + OrderQuantity + '.\n'
get_task_string = ''
get_task_string_with_header_detail = ''
get_task_string = task_title + '.' + '\n'
get_task_string_with_header_detail = 'created by user: '******'.' + '\n' + 'SupplierName: ' + SupplierName \
+ '.' + '\n' + 'PurchaseOrderNetAmount: ' + PurchaseOrderNetAmount + ' ' + DocumentCurrency + '.'+'\n'
# final_reply_string = 'Now you have got, '+ str(no_of_tasks) + ' pending tasks to approve. ' + get_task_string + get_task_string_with_header_detail +'You have: ' + str(no_of_line_items) +' items.\n'+ concat_string_for_multiple_lineitems + " say approve to approve this task or say ignore to skip this task and move on to your next task, or say next to get your next task with details."
final_reply_string = 'Now you have got, ' + str(
no_of_tasks
) + ' pending tasks to approve. ' + get_task_string + get_task_string_with_header_detail + 'You have: ' + str(
no_of_line_items
) + ' items.\n' + " say get item details to get all the item details in this purchase order. Or,say approve to approve this task or say ignore to skip this task and move on to your next task, or say next to get your next task with details."
return final_reply_string, 1, instance_id, created_by_user, SupplierName, (
PurchaseOrderNetAmount + ' ' + DocumentCurrency
), '', all_item_details, no_of_line_items, scrapped_po_no #return 1for memory index as no memo is present in the beggining
elif ((len(body1["d"]["results"]) > 1)
and bot_memo['index'] < len(body1["d"]["results"])):
#task details
instance_id = body1["d"]["results"][
bot_memo['index']]["InstanceID"]
task_title = task_title = body1["d"]["results"][
bot_memo['index']]["TaskTitle"]
#print(task_title)
scrapped_po_no = task_title.split("order ", 1)[1]
#print(scrapped_po_no)
body2, body3 = take_action_async(scrapped_po_no)
#po_header detail
created_by_user = body2["d"]["CreatedByUser"]
SupplierName = body2["d"]["SupplierName"]
PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]
DocumentCurrency = body2["d"]["DocumentCurrency"]
PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]
final_reply_string = ''
concat_string_for_multiple_lineitems = ''
per_item_desc_dict = {}
all_item_details = {}
#po item detail
#only show one or two tasks
no_of_line_items = len(body3["d"]["results"])
for i in range(no_of_line_items):
Material = body3["d"]["results"][i]["Material_Text"]
Plant = body3["d"]["results"][i]["Plant"]
OrderQuantity = body3["d"]["results"][i]["OrderQuantity"]
netPriceItem = body3["d"]["results"][i]["NetPriceAmount"]
documentCurrency = body3["d"]["results"][i]["DocumentCurrency"]
price_present_item_with_currency = netPriceItem + documentCurrency
item_no = 'item : ' + str(i + 1)
# print(item_no)
#item_no = dict(item_no)
per_item_desc_dict = {
item_no: {
'Material': Material,
'Plant': Plant,
'OrderQuantity': OrderQuantity,
'netPriceItem': price_present_item_with_currency
}
}
all_item_details.update(per_item_desc_dict)
#use this when sending the item details as string all in one reply
# concat_string_for_multiple_lineitems = concat_string_for_multiple_lineitems \
# + 'Material: ' + Material + '.\n' + 'plant: ' + Plant + '.\n' \
# + 'OrderQuantity: ' + OrderQuantity + '.\n'
get_task_string = ''
get_task_string_with_header_detail = ''
get_task_string = task_title + '.' + '\n'
get_task_string_with_header_detail = 'created by user: '******'.' + '\n' + 'SupplierName: ' + SupplierName \
+ '.' + '\n' + 'PurchaseOrderNetAmount: ' + PurchaseOrderNetAmount + ' ' + DocumentCurrency + '.'+'\n'
# final_reply_string = get_task_string + get_task_string_with_header_detail +'You have: ' + str(no_of_line_items) +' items in this P.O.\n'+ concat_string_for_multiple_lineitems + " say approve to approve this task or say ignore to skip this task and move on to your next task, or say next to get your next task with details."
final_reply_string = 'Now you have got, ' + str(
no_of_tasks
) + ' pending tasks to approve. ' + get_task_string + get_task_string_with_header_detail + 'You have: ' + str(
no_of_line_items
) + ' items.\n' + " say get item details to get all the item details in this purchase order. Or,say approve to approve this task or say ignore to skip this task and move on to your next task, or say next to get your next task with details."
#print(get_task_string)
#print(final_reply_string)
return final_reply_string, bot_memo[
'index'] + 1, instance_id, created_by_user, SupplierName, (
PurchaseOrderNetAmount + ' ' + DocumentCurrency
), '', all_item_details, no_of_line_items, scrapped_po_no
elif (len(body1["d"]["results"]) > 0) and (bot_memo['index'] >= len(
body1["d"]["results"])):
final_reply_string = 'no more tasks to approve in your inbox.'
return final_reply_string, bot_memo['index'], len(
body1["d"]["results"]
), bot_memo['created_by'], bot_memo['SupplierName'], bot_memo[
'PurchaseOrderNetAmount'], '', '', '', bot_memo[
'scrapped_po_no']
else:
final_reply_string = 'I think there are no more pending approvals for you. Say, "get my tasks", to get your pending approvals.'
return final_reply_string, bot_memo['index'], len(
body1["d"]["results"]
), bot_memo['created_by'], bot_memo['SupplierName'], bot_memo[
'PurchaseOrderNetAmount'], '', '', '', bot_memo[
'scrapped_po_no']
#repeat intent is handled via bot memory not via code
# elif((bot_memo['index']) and present_skill == 'repeat'):
# r = requests.get("https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json", auth=HTTPBasicAuth('pritamsa', 'rupu@0801'))
# body1 = r.json()
# if (body1["d"]["results"] and bot_memo['index'] <= len(body1["d"]["results"])):
# #task details
# instance_id = body1["d"]["results"][bot_memo['index']-1]["InstanceID"]
# task_title = body1["d"]["results"][bot_memo['index']-1]["TaskTitle"]
# scrapped_po_no = task_title.split("order ",1)[1]
# body2,body3 = take_action_async(scrapped_po_no)
# #po_header detail
# created_by_user = body2["d"]["CreatedByUser"]
# SupplierName = body2["d"]["SupplierName"]
# PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]
# DocumentCurrency = body2["d"]["DocumentCurrency"]
# PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]
# final_reply_string = ''
# concat_string_for_multiple_lineitems = ''
# #po item detail
# #only show one or two tasks
# no_of_line_items = len(body3["d"]["results"])
# for i in range(no_of_line_items):
# Material = body3["d"]["results"][i]["Material_Text"]
# Plant = body3["d"]["results"][i]["Plant"]
# OrderQuantity = body3["d"]["results"][i]["OrderQuantity"]
# concat_string_for_multiple_lineitems = concat_string_for_multiple_lineitems \
# + 'Material: ' + Material + '.\n' + 'plant: ' + Plant + '.\n' \
# + 'OrderQuantity: ' + OrderQuantity + '.\n'
# get_task_string = ''
# get_task_string_with_header_detail = ''
# get_task_string = task_title + '\n'
# get_task_string_with_header_detail = 'created_by_user: '******'.' +'\n' + 'SupplierName: ' + SupplierName \
# +'.' + '\n' + 'PurchaseOrderNetAmount: ' + PurchaseOrderNetAmount + ' ' + DocumentCurrency + '.' +'\n'
# final_reply_string = get_task_string + get_task_string_with_header_detail +'You have: ' + str(no_of_line_items) +' items\n'+ concat_string_for_multiple_lineitems + " say approve to approve this task or say ignore to skip this task and move on to your next task, or say next to get your next task with details."
# #print(get_task_string)
# #print(final_reply_string)
# return final_reply_string,bot_memo['index'],instance_id,created_by_user,SupplierName, (PurchaseOrderNetAmount + ' ' + DocumentCurrency)
# elif(body1["d"]["results"] and bot_memo['index'] >= len(body1["d"]["results"])):
# final_reply_string = 'no more tasks to approve...'
# return final_reply_string,bot_memo['index'],len(body1["d"]["results"]),created_by_user,SupplierName, (PurchaseOrderNetAmount + ' ' + DocumentCurrency)
# else:
# final_reply_string = 'I am facing some issues now please try later'
# return final_reply_string,bot_memo['index'],len(body1["d"]["results"]),created_by_user,SupplierName, (PurchaseOrderNetAmount + ' ' + DocumentCurrency)
elif ((bot_memo['index']) and present_skill == 'approve'):
after_approval_reply = 'successfully approved, please say,"get my tasks", to get your previous pending aapprovals from the beggining, or, say next to move on to your next task.'
approval_failure_reply = "there was an issue with the server, Please try again later to approve..."
session = requests.Session()
header = {'x-csrf-token': 'Fetch'}
present_task_instance_id = bot_memo['instanceID']
# response = session.head("https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json", auth=HTTPBasicAuth('pritamsa', 'rupu@0801'),headers=header)
# if (response.status_code != 200):
# return approval_failure_reply ,bot_memo['index'],present_task_instance_id,bot_memo['created_by'],bot_memo['SupplierName'], bot_memo['PurchaseOrderNetAmount'],approval_failure_reply,'','',bot_memo['scrapped_po_no']
# elif (response.status_code == 200):
# cookie = session.cookies.get_dict()
# print(cookie)
# csrf = response.headers['x-csrf-token']
# #print(csrf)
# #post
# #approve
# header_2 = {'x-csrf-token':csrf}
# approve_po = session.post("https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/Decision?sap-client=400&SAP__Origin='S4HMYINBOCLNT200'&InstanceID="+ "'"+present_task_instance_id +"'""&DecisionKey='0001'&Comments='test%20approve'",auth=HTTPBasicAuth('pritamsa', 'rupu@0801'),headers=header_2,cookies=cookie)
# print('***************************************************************')
# print(approve_po.status_code)
# approval request posted asynchronously
url3 = [
"https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json"
]
head_res1 = (grequests.head(u,
auth=('pritamsa', 'rupu@0801'),
headers=header) for u in url3)
#both imap and map can be used
#reque = grequests.imap(rs,size=1)
reque3 = grequests.map(head_res1, size=1)
response_array3 = []
for response3 in reque3:
if (response3.status_code != 200):
print("hey problem")
return approval_failure_reply, bot_memo[
'index'], present_task_instance_id, bot_memo[
'created_by'], bot_memo['SupplierName'], bot_memo[
'PurchaseOrderNetAmount'], approval_failure_reply, '', '', bot_memo[
'scrapped_po_no']
else:
cookie = response3.cookies.get_dict()
print(cookie)
csrf = response3.headers['x-csrf-token']
print(csrf)
header_2 = {'x-csrf-token': csrf}
url_post = [
"https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/Decision?sap-client=400&SAP__Origin='S4HMYINBOCLNT200'&InstanceID="
+ "'" + present_task_instance_id + "'"
"&DecisionKey='0001'&Comments='test%20approve'"
]
post_res = (grequests.post(u_post,
auth=('pritamsa', 'rupu@0801'),
headers=header_2,
cookies=cookie)
for u_post in url_post)
post_reque = grequests.map(post_res, size=1)
response_array_post = []
for response_post in post_reque:
if (response_post.status_code != 200):
print(
"hey problem in approving the request. Please try again later."
)
return approval_failure_reply, bot_memo[
'index'], present_task_instance_id, bot_memo[
'created_by'], bot_memo['SupplierName'], bot_memo[
'PurchaseOrderNetAmount'], approval_failure_reply, '', '', bot_memo[
'scrapped_po_no']
else:
return after_approval_reply, bot_memo[
'index'], present_task_instance_id, bot_memo[
'created_by'], bot_memo['SupplierName'], bot_memo[
'PurchaseOrderNetAmount'], after_approval_reply, '', '', bot_memo[
'scrapped_po_no'] #after this call the "next" task showing skill in bot
elif ((bot_memo['index']) and present_skill == 'reject'):
after_rejection_reply = 'successfully rejected, please say,"get my tasks", to get your previous pending aapprovals from the beggining, or, say next to move on to your next task.'
rejection_failure_reply = "there was an issue with the server, Please try again later to approve..."
session = requests.Session()
header = {'x-csrf-token': 'Fetch'}
present_task_instance_id = bot_memo['instanceID']
# response = session.head("https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json", auth=HTTPBasicAuth('pritamsa', 'rupu@0801'),headers=header)
# if (response.status_code != 200):
# return approval_failure_reply ,bot_memo['index'],present_task_instance_id,bot_memo['created_by'],bot_memo['SupplierName'], bot_memo['PurchaseOrderNetAmount'],approval_failure_reply,'','',bot_memo['scrapped_po_no']
# elif (response.status_code == 200):
# cookie = session.cookies.get_dict()
# print(cookie)
# csrf = response.headers['x-csrf-token']
# #print(csrf)
# #post
# #approve
# header_2 = {'x-csrf-token':csrf}
# approve_po = session.post("https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/Decision?sap-client=400&SAP__Origin='S4HMYINBOCLNT200'&InstanceID="+ "'"+present_task_instance_id +"'""&DecisionKey='0001'&Comments='test%20approve'",auth=HTTPBasicAuth('pritamsa', 'rupu@0801'),headers=header_2,cookies=cookie)
# print('***************************************************************')
# print(approve_po.status_code)
# approval request posted asynchronously
url4 = [
"https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json"
]
head_res4 = (grequests.head(u,
auth=('pritamsa', 'rupu@0801'),
headers=header) for u in url4)
#both imap and map can be used
#reque = grequests.imap(rs,size=1)
reque4 = grequests.map(head_res4, size=1)
response_array4 = []
for response4 in reque4:
if (response4.status_code != 200):
print("hey problem")
return rejection_failure_reply, bot_memo[
'index'], present_task_instance_id, bot_memo[
'created_by'], bot_memo['SupplierName'], bot_memo[
'PurchaseOrderNetAmount'], rejection_failure_reply, '', '', bot_memo[
'scrapped_po_no']
else:
cookie = response4.cookies.get_dict()
print(cookie)
csrf = response4.headers['x-csrf-token']
print(csrf)
header_2 = {'x-csrf-token': csrf}
url_post = [
"https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/Decision?sap-client=400&SAP__Origin='S4HMYINBOCLNT200'&InstanceID="
+ "'" + present_task_instance_id + "'"
"&DecisionKey='0002'&Comments='test%20reject'"
]
post_res = (grequests.post(u_post,
auth=('pritamsa', 'rupu@0801'),
headers=header_2,
cookies=cookie)
for u_post in url_post)
post_reque = grequests.map(post_res, size=1)
response_array_post = []
for response_post in post_reque:
if (response_post.status_code != 200):
print(
"hey problem in rejecting P.O. . Please try again later."
)
return rejection_failure_reply, bot_memo[
'index'], present_task_instance_id, bot_memo[
'created_by'], bot_memo['SupplierName'], bot_memo[
'PurchaseOrderNetAmount'], rejection_failure_reply, '', '', bot_memo[
'scrapped_po_no']
else:
return after_rejection_reply, bot_memo[
'index'], present_task_instance_id, bot_memo[
'created_by'], bot_memo['SupplierName'], bot_memo[
'PurchaseOrderNetAmount'], after_rejection_reply, '', '', bot_memo[
'scrapped_po_no'] #after this call the "next" task showing skill in bot
# THIS LOGIC BELOW NEEDS TO BE RE_WRITTEN
#************************************************************************************************************
# elif((bot_nlp['ordinal'] and len(bot_nlp['ordinal']) <= bot_memo['no_of_line_items']) and present_skill == 'get_item_details'):
elif (present_skill == 'get_item_details'):
if (bot_nlp['ordinal']
and len(bot_nlp['ordinal']) <= bot_memo['no_of_line_items']):
# filter_item_ordinally = 'item : '+ (bot_nlp['ordinal'][bot_nlp['ordinal']['index']]['rank'])
# print(filter_item_ordinally)
print('///////////////////////////////////////////////////')
nlp_ordinal_filter_index = bot_nlp['ordinal'][0][
'index'] #this is the first element's index of nlp entity ordinal array
individual_item_filter_string = 'item : ' + str(
nlp_ordinal_filter_index + 1)
item_level_reply_ordinally = bot_memo['all_item_details'][
individual_item_filter_string]
print(item_level_reply_ordinally)
return str(item_level_reply_ordinally).strip(
'{}'
), bot_memo['index'], bot_memo['instanceID'], bot_memo[
'created_by'], bot_memo['SupplierName'], bot_memo[
'PurchaseOrderNetAmount'], bot_memo[
'after_approval_reply'], bot_memo[
'all_item_details'], bot_memo[
'no_of_line_items'], bot_memo['scrapped_po_no']
elif (bot_nlp['ordinal'] == False and bot_nlp['number']
and len(bot_nlp['number']) <= bot_memo['no_of_line_items']):
# filter_item_ordinally = 'item : '+ (bot_nlp['ordinal'][bot_nlp['ordinal']['index']]['rank'])
# print(filter_item_ordinally)
print('///////////////////////////////////////////////////')
nlp_number_filter_index = bot_nlp['number'][0][
'scalar'] #this is the first element's index of nlp entity ordinal array
individual_item_filter_string = 'item : ' + str(
nlp_number_filter_index)
item_level_reply_numerically = bot_memo['all_item_details'][
individual_item_filter_string]
print(item_level_reply_numerically)
return str(
item_level_reply_numerically
), bot_memo['index'], bot_memo['instanceID'], bot_memo[
'created_by'], bot_memo['SupplierName'], bot_memo[
'PurchaseOrderNetAmount'], bot_memo[
'after_approval_reply'], bot_memo[
'all_item_details'], bot_memo[
'no_of_line_items'], bot_memo['scrapped_po_no']
def convert(urls2, pf, pft, etfnames):
# Start empty list of ETF holdings
etf_lib = list()
# Async requests
reqs = [grequests.get(url) for url in urls2]
resp = grequests.map(reqs)
# Parse data tables with pandas and append data to return variable
for i, r in enumerate(resp):
get_text = r.text
try:
wds = pd.read_html(get_text)[1]
# This statement skips equities so the scraper can pull ETF data only
except:
pass
else:
wds = wds.rename(columns={
"%\xa0Weight": "% Weight",
"%\xa0Change": "% Chg"
})
# Filter col of interest and convert '% Weight' col from str to float, format Symbol col
for j in range(0, len(wds['% Weight'])):
wds.at[j, '% Weight'] = wds.at[j, '% Weight'].replace("%", "")
wds.at[j, 'Symbol'] = wds.at[j, 'Symbol'].replace("-", ".")
wds['% Weight'] = wds['% Weight'].astype(float)
# Delete unused data
del wds['Price']
del wds['% Chg']
# Create MISC ticker which represents the % holding of the ETF not accouted for by top 25 holdings
etft = 100 - wds['% Weight'].sum()
new_row = {'Symbol': (pf.at[i, 'Symbol']), '% Weight': etft}
wds = wds.append(new_row, ignore_index=True)
# Multiply ETF ticker list by weight in portfolio
wds['% Weight'] = wds['% Weight'] * pf.at[i, '% Weight']
# Append to list of ETF data and remove ETF ticker from list and remove original ticker entry after it has been parsed
etf_lib.append(wds)
pf.drop([i], inplace=True)
# Concatenate lists together and sum any repeat tickers in list
df = pd.concat(etf_lib)
pf['% Weight'] *= 100
df = df.append(pf)
# Save names as a dict object with symbols as keys and names as values
names = dict(zip(df['Symbol'], df['Name']))
# This command will combine repeat tickers and sum their values, but doing so deletes the Name col
df = df.groupby(['Symbol'], as_index=False).sum()
out = df.sort_values(by='% Weight', ascending=False)
# Add empty Name col to out df and re-index
out["Name"] = ['' for _ in range(len(out))]
columnsTitles = ['Symbol', 'Name', '% Weight', 'Total Value']
out = out.reindex(columns=columnsTitles)
# Correct name column to values saved in names dictionary
for i in range(0, len(out['Symbol'])):
for j in names.keys():
if str(j) == str(out.at[i, 'Symbol']):
out.at[i, 'Name'] = names.get(j)
# Re-add ETF Names
for i in range(0, len(out['Symbol'])):
for j in etfnames.keys():
if str(j) in str(out.at[i, 'Symbol']):
out.at[i,
'Name'] = ('Misc ' + str(etfnames.get(j)) + ' Holdings')
# Update and format before print out
val = pft * (out['% Weight'] / 100)
out['Total Value'] = val
out['Total Value'] = out['Total Value'].round(decimals=2)
out['% Weight'] = out['% Weight'].round(decimals=3)
# Return result
return out
def fetchRaceAndGenderFor(cls, people):
peopleCopy = list(people)
# Get the search page for each person
urls = list()
for person in peopleCopy:
url = "http://search.nndb.com/search/?type=unspecified&query="
url += "+".join(person.name.split())
urls.append(url)
requests = [grequests.get(url) for url in urls]
responses = grequests.map(requests)
# Get the profile pages from each search page
urls = []
notFoundIndices = set()
for i in range(len(responses)):
response = responses[i]
if not response:
notFoundIndices.add(i)
continue
name = peopleCopy[i].name.split()
if len(name) == 0:
notFoundIndices.add(i)
continue
searchTree = html.fromstring(response.content)
condition = '[contains(.//text(), "%s")' % name[0]
if len(name) > 1:
condition += ' and contains(.//text(), "%s")' % name[1]
condition += ']'
personList = searchTree.xpath('//table')[3].xpath('./tr')
if len(personList) <= 1:
notFoundIndices.add(i)
continue
personList = personList[1].xpath('./td' + condition)
if not personList or len(personList[0].xpath('.//a/@href')) == 0:
notFoundIndices.add(i)
continue
urls.append(personList[0].xpath('.//a/@href')[0])
# remove the people that we couldn't find info for
for i in sorted(list(notFoundIndices), reverse=True):
del peopleCopy[i]
requests = [grequests.get(url) for url in urls]
responses = grequests.map(requests)
# Go through responses and scrape race and gender info
for i in range(len(responses)):
response = responses[i]
person = peopleCopy[i]
personTree = html.fromstring(response.content)
raceStr = '//p/b[text()="Race or Ethnicity:"]'
raceStr += '/following-sibling::text()'
if len(personTree.xpath(raceStr)) > 0:
person.race = personTree.xpath(raceStr)[0].strip()
genderStr = '//p/b[text()="Gender:"]/following-sibling::text()'
if len(personTree.xpath(genderStr)) > 0:
person.gender = personTree.xpath(genderStr)[0].strip()
def main(file, url, async, verbose):
f = open(file, 'r')
num_lines = file_len(file)
start_time = time.time()
count = 0
if verbose:
print "[+] Testing URL's ".format(num_lines)
urls = []
for line in f:
line = line.rstrip('\n')
formated_url = url.format(line)
urls.append(formated_url)
if async:
rs = (grequests.get(u) for u in urls)
responses = grequests.map(rs)
for response in responses:
if response.status_code != 404:
print response.url + " : " + str(response.status_code)
count += 1
else:
for url in urls:
r = requests.get(url, verify=False)
if r.status_code != 404:
print "[*] " + r.url + " : " + str(r.status_code)
count += 1
else if response.status_code == '401' or == '403':
print response.url + "Access Denied or unauthorized access"
count += 1
urls.append(b)
def my_exception_handler(req, e):
print(req)
print(e)
print(dir(req))
print(dir(e))
MAX_CONNECTIONS = 50 # Number of connections you want to limit it to
print(1111111)
pages = len(urls)
for i in range(1, pages + 1, MAX_CONNECTIONS):
print("1 Waiting %s" % i) # Optional, so you see something is done.
rs = (grequests.get(u, timeout=100, verify=False)
for u in urls[i:i + MAX_CONNECTIONS])
a = list(rs)
time.sleep(0.2) # You can change this to whatever you see works better.
results = grequests.map(
a, exception_handler=my_exception_handler
) # The key here is to extend, not append, not insert.
print("result1 : %s" % len(results))
for x in results:
if x:
print(x.status_code)
try:
soup = BeautifulSoup(x.text, 'html.parser')
for link in soup.find_all('a'):
if link.get('href'):
if link.get('href').find('/providers/profile/') != -1:
if args.nofaces:
sub_names.remove("faces")
if args.noflags:
sub_names.remove("flags")
if args.nosublist:
sub_names.remove("sublist")
if args.html:
file_names.append("Ranking.html")
if args.css:
file_names.append("Ranking.css")
print("Saving directories...")
rs = [
grequests.get(url + "/" + dir_names[i] + "/",
verify=False,
stream=False,
session=sessions[i % NUM_SESSIONS])
for i in range(len(dir_names))
]
dir_reqs = grequests.map(rs)
output = args.output
if output is None:
output = list(dir_reqs[dir_names.index("contests")].json().keys())[0]
print("Copying static files...")
copy_tree("cmsranking", output, update=1)
for i in range(len(dir_names)):
if dir_reqs[i] is not None:
os.makedirs(output + "/" + dir_names[i], exist_ok=True)
def getQueries():
O0OO0O0O00O0OOO00 = ''
OO0O0O0O00OO0000O = ''
O0O00OOOOOOO0O000 = ''
O0OOOOOOOO0OO0O0O = ''
OO0OOOOO0O0O0OOOO = ''
OO00000OO0OOO0OOO = ''
OOOO0O0O00OOO0O00 = ''
OO0O0O000O00O0O00 = ''
OO000OOOO0O0000O0 = ''
OOO0O0OOOO0O0O00O = ''
OOO00O00OO0O00O0O = ''
OO0O000O00OOOO0O0 = ''
O0OOO00OOO00OO00O = ''
O0OOO000000OOOOOO = ''
OO00OOO000O00OO00 = ''
O0OO0OOO0O0OO0000 = ''
OO00O000OO000OO0O = ''
O00O000OO000O00O0 = ''
O000O0O0O0OO0O00O = ''
with open('pairs.txt', 'rb') as (O0OO0OO0000O000OO):
O0OO00OO000OO0O00 = OOOO00OOOO00O000O.load(O0OO0OO0000O000OO)
O000OOO0OOOO0O0OO = 0
for OO0O0O0O0O0O00O00 in O0OO00OO000OO0O00:
if O000OOO0OOOO0O0OO <= 50:
O0OO0O0O00O0OOO00 += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 50 and O000OOO0OOOO0O0OO <= 100:
OO0O0O0O00OO0000O += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 100 and O000OOO0OOOO0O0OO <= 150:
O0O00OOOOOOO0O000 += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 150 and O000OOO0OOOO0O0OO <= 200:
O0OOOOOOOO0OO0O0O += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 200 and O000OOO0OOOO0O0OO <= 250:
OO0OOOOO0O0O0OOOO += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 250 and O000OOO0OOOO0O0OO <= 300:
OO00000OO0OOO0OOO += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 300 and O000OOO0OOOO0O0OO <= 350:
OOOO0O0O00OOO0O00 += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 350 and O000OOO0OOOO0O0OO <= 400:
OO0O0O000O00O0O00 += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 400 and O000OOO0OOOO0O0OO <= 450:
OO000OOOO0O0000O0 += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 450 and O000OOO0OOOO0O0OO <= 500:
OOO0O0OOOO0O0O00O += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 500 and O000OOO0OOOO0O0OO <= 550:
OOO00O00OO0O00O0O += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 550 and O000OOO0OOOO0O0OO <= 600:
OO0O000O00OOOO0O0 += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 600 and O000OOO0OOOO0O0OO <= 650:
O0OOO00OOO00OO00O += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 650 and O000OOO0OOOO0O0OO <= 700:
O0OOO000000OOOOOO += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 700 and O000OOO0OOOO0O0OO <= 750:
OO00OOO000O00OO00 += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 750 and O000OOO0OOOO0O0OO <= 800:
O0OO0OOO0O0OO0000 += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 800 and O000OOO0OOOO0O0OO <= 850:
OO00O000OO000OO0O += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 850 and O000OOO0OOOO0O0OO <= 900:
O00O000OO000O00O0 += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
if O000OOO0OOOO0O0OO > 900 and O000OOO0OOOO0O0OO <= 950:
O000O0O0O0OO0O00O += str(OO0O0O0O0O0O00O00) + '-'
O000OOO0OOOO0O0OO += 1
querys.append(O0OO0O0O00O0OOO00[:-1])
querys.append(OO0O0O0O00OO0000O[:-1])
querys.append(O0O00OOOOOOO0O000[:-1])
querys.append(O0OOOOOOOO0OO0O0O[:-1])
querys.append(OO0OOOOO0O0O0OOOO[:-1])
querys.append(OO00000OO0OOO0OOO[:-1])
querys.append(OOOO0O0O00OOO0O00[:-1])
querys.append(OO0O0O000O00O0O00[:-1])
querys.append(OO000OOOO0O0000O0[:-1])
querys.append(OOO0O0OOOO0O0O00O[:-1])
querys.append(OOO00O00OO0O00O0O[:-1])
querys.append(OO0O000O00OOOO0O0[:-1])
querys.append(O0OOO00OOO00OO00O[:-1])
querys.append(O0OOO000000OOOOOO[:-1])
querys.append(OO00OOO000O00OO00[:-1])
O0O000O0OO00000OO = 'https://yobit.net/api/3/ticker/'
O0O0O00O000O0OOOO = (OOO0O00O0O000OO00.get(
O0O000O0OO00000OO + OOO00OO0OO0O00OOO,
headers={
'apisign':
OOO00OO000O000000.new(secret.encode(), OOO00OO0OO0O00OOO.encode(),
OO0OOO000000OO000.sha512).hexdigest()
}) for OOO00OO0OO0O00OOO in querys)
OOO0OOO0OO000000O = OOO0O00O0O000OO00.map(O0O0O00O000O0OOOO)
O0O0000O0O0OOOOOO = [
O00OOO0O00OOOOO0O.json() for O00OOO0O00OOOOO0O in OOO0OOO0OO000000O
]
for O000O000O0000O0OO in O0O0000O0O0OOOOOO:
for OO0O00OOO00O00000 in O000O000O0000O0OO:
O0O0000O0O00OO000 = OO0O00OOO00O00000
volume.append(O000O000O0000O0OO[O0O0000O0O00OO000]['vol'])
price.append(O000O000O0000O0OO[O0O0000O0O00OO000]['last'])
names.append(O0O0000O0O00OO000)
def main():
global hashes
hashes = set()
past_urls = set()
now = datetime.datetime.now()
parser = argparse.ArgumentParser()
parser.add_argument("-p",
"--proxy",
help="Define HTTP proxy as address:port")
parser.add_argument("-d",
"--dumpdir",
help="Define dump directory for retrieved files")
parser.add_argument("-l",
"--logfile",
help="Define file for logging progress")
parser.add_argument("-x",
"--vxcage",
help="Dump the files to a VxCage instance",
action="store_true",
default=False)
parser.add_argument("-v",
"--viper",
help="Dump the files to a Viper instance",
action="store_true",
default=False)
parser.add_argument("-c",
"--cuckoo",
help="Enable Cuckoo analysis",
action="store_true",
default=False)
parser.add_argument("-s",
"--sort_mime",
help="Sort files by MIME type",
action="store_true",
default=False)
global cfg
cfg = dict()
args = parser.parse_args()
global config
config = ConfigParser.ConfigParser()
config.read('maltrieve.cfg')
if args.logfile or config.get('Maltrieve', 'logfile'):
if args.logfile:
cfg['logfile'] = args.logfile
else:
cfg['logfile'] = config.get('Maltrieve', 'logfile')
logging.basicConfig(filename=cfg['logfile'],
level=logging.DEBUG,
format='%(asctime)s %(thread)d %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
else:
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(thread)d %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
if args.proxy:
cfg['proxy'] = {'http': args.proxy}
elif config.has_option('Maltrieve', 'proxy'):
cfg['proxy'] = {'http': config.get('Maltrieve', 'proxy')}
else:
cfg['proxy'] = None
if config.has_option('Maltrieve', 'User-Agent'):
cfg['User-Agent'] = {
'User-Agent': config.get('Maltrieve', 'User-Agent')
}
else:
cfg['User-Agent'] = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)"
cfg['sort_mime'] = args.sort_mime
if cfg['proxy']:
logging.info('Using proxy %s', cfg['proxy'])
my_ip = requests.get('http://ipinfo.io/ip', proxies=cfg['proxy']).text
logging.info('External sites see %s', my_ip)
print "External sites see %s" % my_ip
cfg['vxcage'] = args.vxcage or config.has_option('Maltrieve', 'vxcage')
cfg['cuckoo'] = args.cuckoo or config.has_option('Maltrieve', 'cuckoo')
cfg['viper'] = args.viper or config.has_option('Maltrieve', 'viper')
cfg['logheaders'] = config.get('Maltrieve', 'logheaders')
black_list = []
if config.has_option('Maltrieve', 'black_list'):
black_list = config.get('Maltrieve', 'black_list').strip().split(',')
white_list = False
if config.has_option('Maltrieve', 'white_list'):
white_list = config.get('Maltrieve', 'white_list').strip().split(',')
# make sure we can open the directory for writing
if args.dumpdir:
cfg['dumpdir'] = args.dumpdir
elif config.get('Maltrieve', 'dumpdir'):
cfg['dumpdir'] = config.get('Maltrieve', 'dumpdir')
else:
cfg['dumpdir'] = '/tmp/malware'
# Create the dir
if not os.path.exists(cfg['dumpdir']):
os.makedirs(cfg['dumpdir'])
try:
d = tempfile.mkdtemp(dir=cfg['dumpdir'])
except Exception as e:
logging.error('Could not open %s for writing (%s), using default',
cfg['dumpdir'], e)
cfg['dumpdir'] = '/tmp/malware'
else:
os.rmdir(d)
logging.info('Using %s as dump directory', cfg['dumpdir'])
if os.path.exists('hashes.json'):
with open('hashes.json', 'rb') as hashfile:
hashes = json.load(hashfile)
elif os.path.exists('hashes.obj'):
with open('hashes.obj', 'rb') as hashfile:
hashes = pickle.load(hashfile)
if os.path.exists('urls.json'):
with open('urls.json', 'rb') as urlfile:
past_urls = json.load(urlfile)
elif os.path.exists('urls.obj'):
with open('urls.obj', 'rb') as urlfile:
past_urls = pickle.load(urlfile)
print "Processing source URLs"
source_urls = {
'https://zeustracker.abuse.ch/monitor.php?urlfeed=binaries':
process_xml_list_desc,
'http://www.malwaredomainlist.com/hostslist/mdl.xml':
process_xml_list_desc,
'http://malc0de.com/rss/': process_xml_list_desc,
'http://vxvault.siri-urz.net/URL_List.php': process_simple_list,
'http://urlquery.net/': process_urlquery,
'http://support.clean-mx.de/clean-mx/rss?scope=viruses&limit=0%2C64':
process_xml_list_title,
'http://malwareurls.joxeankoret.com/normal.txt': process_simple_list
}
headers = {'User-Agent': 'Maltrieve'}
reqs = [
grequests.get(url, timeout=60, headers=headers, proxies=cfg['proxy'])
for url in source_urls
]
source_lists = grequests.map(reqs)
print "Completed source processing"
headers['User-Agent'] = cfg['User-Agent']
malware_urls = set()
for response in source_lists:
if hasattr(response, 'status_code') and response.status_code == 200:
malware_urls.update(source_urls[response.url](response.text))
print "Downloading samples, check log for details"
malware_urls -= past_urls
reqs = [
grequests.get(url, headers=headers, proxies=cfg['proxy'])
for url in malware_urls
]
for chunk in chunker(reqs, 32):
malware_downloads = grequests.map(chunk)
for each in malware_downloads:
if not each or each.status_code != 200:
continue
md5 = save_malware(each, cfg['dumpdir'], black_list, white_list)
if not md5:
continue
past_urls.add(each.url)
print "Completed downloads"
if past_urls:
logging.info('Dumping past URLs to file')
with open('urls.json', 'w') as urlfile:
json.dump(past_urls, urlfile)
if hashes:
with open('hashes.json', 'w') as hashfile:
json.dump(hashes, hashfile)
def request_cpu_load_from_nodes():
# ******************************
# Get Values from Hypervisors
# ******************************
global urls_hypervisors
rs = (grequests.get(u, timeout=TIMEOUT) for u in urls_hypervisors)
results = grequests.map(rs)
if len(results) >= 0:
if results[0] is not None:
push_value_to_list(
data_cpuload_time1,
loads(results[0].content.decode('utf-8'))['total'])
if results[1] is not None:
push_value_to_list(
data_cpuload_time2,
loads(results[1].content.decode('utf-8'))['total'])
if results[2] is not None:
push_value_to_list(
data_cpuload_time3,
loads(results[2].content.decode('utf-8'))['total'])
if results[3] is not None:
push_value_to_list(
data_cpuload_time4,
loads(results[3].content.decode('utf-8'))['total'])
# ******************************
# Get Values from FortiGates
# ******************************
global fgt_sessions
global urls_fgt
fgt_login_requests = [None] * len(urls_fgt)
fgt_cpu_requests = [None] * len(urls_fgt)
# First, request CPU data
for i in range(len(fgt_sessions)):
fgt_cpu_requests[i] = grequests.get(
urls_fgt[i] +
'api/v2/monitor/system/resource/usage?resource=cpu&interval=1-min',
session=fgt_sessions[i],
headers=fgt_sessions[i].headers,
timeout=TIMEOUT,
verify=False)
fgt_cpu_results = grequests.map(fgt_cpu_requests)
# Check if request failed because of login
# If failed, then login
print("fgt_cpu_results:", fgt_cpu_results)
reqs = []
for i in range(len(fgt_sessions)):
if fgt_cpu_results[i] is not None and fgt_cpu_results[
i].status_code == 401:
print("Login into FortiGate's REST API: ", i)
fgt_login_requests[i] = grequests.post(
urls_fgt[i] + 'logincheck',
data='username='******'&secretkey=' + PASSWORD_FGT +
'&ajax=1',
session=fgt_sessions[i],
timeout=TIMEOUT,
verify=False)
r = grequests.send(fgt_login_requests[i])
reqs.append(r)
gevent.joinall(reqs)
# Only if request to get CPU was 200 OK then
# get the value and push it to the list
for i in range(len(fgt_cpu_results)):
if fgt_cpu_results[i] and fgt_cpu_results[i].status_code == 200:
try:
push_value_to_list(
globals()['data_fgtload_time' + str(i + 1)],
loads(fgt_cpu_results[i].content.decode('utf-8'))
['results']['cpu'][0]['current'])
except:
print("Error getting data from FortiGate:", i)
else:
print("FGT request was not ok:", i)
if fgt_cpu_results[i] is not None:
print(" -> result: ", fgt_cpu_results[i].status_code)
push_value_to_list(globals()['data_fgtload_time' + str(i + 1)], -1)
# ********************************
# Get Values from DSO CyberMapper
# ********************************
global url_cybermapper
# Get dpid
loadbal = requests.get(url_cybermapper + '/v1.0/loadbal', timeout=TIMEOUT)
# Use this notation '[*' to get the keys extracted into a list
dpid = [*loads(loadbal.content.decode('utf-8')).keys()][0]
# Get port statistics
results = requests.get(url_cybermapper + '/v1.0/switch_stats/switches/' +
dpid + '/port_stats',
timeout=TIMEOUT)
port_stats = loads(results.content.decode('utf-8'))
bps = {}
for port in port_stats:
bps[port['id']] = (port['tx_bytes'] - port['last']['tx_bytes'] +
port['rx_bytes'] - port['last']['rx_bytes']) / \
(port['timestamp'] - port['last']['timestamp'])
# Instead of port3 (which is faulty) we use 21-24
# Instead of port4 we use 25-28
push_value_to_list(data_totalthroughput_ingress_time,
(bps[1] + bps[21] + bps[22] + bps[23] + bps[24]) /
1000000000 * 8)
push_value_to_list(data_totalthroughput_egress_time,
(bps[2] + bps[25] + bps[26] + bps[27] + bps[28]) /
1000000000 * 8)
push_value_to_list(data_fgtthroughput1_time,
(bps[5] + bps[6]) / 2000000000 * 8)
push_value_to_list(data_fgtthroughput2_time,
(bps[7] + bps[8]) / 2000000000 * 8)
push_value_to_list(data_fgtthroughput3_time,
(bps[9] + bps[10]) / 2000000000 * 8)
push_value_to_list(data_fgtthroughput4_time,
(bps[11] + bps[12]) / 2000000000 * 8)
push_value_to_list(data_fgtthroughput5_time,
(bps[13] + bps[14]) / 2000000000 * 8)
push_value_to_list(data_fgtthroughput6_time,
(bps[15] + bps[16]) / 2000000000 * 8)
def main():
tstUrl = '/nekretnine/prodaja-savski-gaj-neadaptirana-kuca-150-m2-garaza-28m2-100.000-oglas-18694996'
tstId = 18694996
print 'start MAIN'
global exportFeatures
global oglasData
#Params
class Settings:
None
Settings.baseURL = 'http://www.njuskalo.hr'
Settings.startURL = 'http://www.njuskalo.hr/prodaja-kuca/zagreb'
Settings.startURL = 'http://www.njuskalo.hr/prodaja-kuca/zagreb?page=%i'
addRequestss = []
pageRequestss = (grequests.get(Settings.startURL % pageNum)
for pageNum in range(1, 3)) # 1-156
#print pageRequestss
pageResponses = grequests.map(pageRequestss)
#print pageResponses
#print 'start pageResponse LOOP'
for pageResponse in pageResponses:
tree = html.fromstring(pageResponse.content)
# get oglasi
oglasi = tree.xpath(
'//*[@id="form_browse_detailed_search"]/div/div[1]/div[2]/div[4]/ul/li[@class!="EntityList-item--banner"]'
)
#//*[@id="form_browse_detailed_search"]/div/div[1]/div[2]/div[4]/ul/li[1]/article/h3/a
for oglas in oglasi:
dataAddId = oglas.xpath('@data-ad-id')
#print dataAddId
if dataAddId:
dataAddId = [tstId]
#dataAddId = dataAddId[0]
addTitle = oglas.xpath('article/h3/a/text()')
#print addTitle
addUrl = tstUrl #oglas.xpath('article/h3/a/@href')[0]
#print addUrl
#print Settings.baseURL + addUrl
oglasData[(Settings.baseURL + addUrl)] = {
'dataAddId': dataAddId[0],
'addTitle': addTitle[0],
'addUrl': (Settings.baseURL + addUrl)
}
addRequestss.append(
grequests.get(Settings.baseURL + addUrl,
hooks={'response': addProcess}))
#print 'END pageResponse LOOP %s' % pageResponse
# get individual adds
#print addRequestss
addResponses = grequests.map(addRequestss)
#print addResponses
featuresGeoJson = FeatureCollection(exportFeatures)
f = open('data_dump_20161022.geojson', 'w')
f.write(str(featuresGeoJson)) # python will convert \n to os.linesep
f.close()
def test_query_partners_by_token(
api_backend,
api_test_context,
api_raiden_service):
# let's create 2 new channels for the same token
first_partner_address = '0x61c808d82a3ac53231750dadc13c777b59310bd9'
second_partner_address = '0x29fa6cf0cce24582a9b20db94be4b6e017896038'
token_address = '0xea674fdde714fd979de3edf0f56aa9716b898ec8'
settle_timeout = 1650
channel_data_obj = {
'partner_address': first_partner_address,
'token_address': token_address,
'settle_timeout': settle_timeout
}
request = grequests.put(
api_url_for(api_backend, 'channelsresource'),
json=channel_data_obj
)
response = request.send().response
assert_proper_response(response)
response = response.json()
first_channel_address = response['channel_address']
channel_data_obj['partner_address'] = second_partner_address
request = grequests.put(
api_url_for(api_backend, 'channelsresource'),
json=channel_data_obj,
)
response = request.send().response
assert_proper_response(response)
response = response.json()
second_channel_address = response['channel_address']
# and a channel for another token
channel_data_obj['partner_address'] = '0xb07937AbA15304FBBB0Bf6454a9377a76E3dD39E'
channel_data_obj['token_address'] = '0x70faa28A6B8d6829a4b1E649d26eC9a2a39ba413'
request = grequests.put(
api_url_for(api_backend, 'channelsresource'),
json=channel_data_obj
)
response = request.send().response
assert_proper_response(response)
# and now let's query our partners per token for the first token
request = grequests.get(
api_url_for(
api_backend,
'partnersresourcebytokenaddress',
token_address=token_address,
)
)
response = request.send().response
assert_proper_response(response)
response = response.json()
expected_response = [
{
'partner_address': first_partner_address,
'channel': '/api/1/channels/{}'.format(first_channel_address)
}, {
'partner_address': second_partner_address,
'channel': '/api/1/channels/{}'.format(second_channel_address)
}
]
assert all(r in response for r in expected_response)
def test_connect_and_leave_token_network(
api_backend,
api_test_context,
api_raiden_service):
# first check we don't have any open channels
request = grequests.get(
api_url_for(api_backend, 'channelsresource')
)
response = request.send().response
assert_proper_response(response)
channels = response.json()
assert not channels
assert response.json() == api_test_context.expect_channels()
funds = 100
initial_channel_target = DEFAULT_INITIAL_CHANNEL_TARGET
joinable_funds_target = DEFAULT_JOINABLE_FUNDS_TARGET
token_address = '0xea674fdde714fd979de3edf0f56aa9716b898ec8'
connect_data_obj = {
'funds': funds,
}
request = grequests.put(
api_url_for(api_backend, 'connectionsresource', token_address=token_address),
json=connect_data_obj,
)
response = request.send().response
assert_proper_response(response)
# check that channels got created
request = grequests.get(
api_url_for(api_backend, 'channelsresource')
)
response = request.send().response
assert_proper_response(response)
channels = response.json()
# There should be three channels according to the default initial_channel_target
assert len(channels) == DEFAULT_INITIAL_CHANNEL_TARGET
assert response.json() == api_test_context.expect_channels()
expected_balance = int((funds * joinable_funds_target) / initial_channel_target)
assert channels[0]['balance'] == expected_balance
assert channels[1]['balance'] == expected_balance
assert channels[2]['balance'] == expected_balance
assert channels[0]['state'] == CHANNEL_STATE_OPENED
assert channels[1]['state'] == CHANNEL_STATE_OPENED
assert channels[2]['state'] == CHANNEL_STATE_OPENED
# Let's leave the token network
request = grequests.delete(
api_url_for(api_backend, 'connectionsresource', token_address=token_address),
)
response = request.send().response
assert_proper_response(response)
# check that all channels were settled after calling `leave`
request = grequests.get(
api_url_for(api_backend, 'channelsresource')
)
response = request.send().response
assert_proper_response(response)
channels = response.json()
assert channels[0]['state'] == CHANNEL_STATE_SETTLED
assert channels[1]['state'] == CHANNEL_STATE_SETTLED
assert channels[2]['state'] == CHANNEL_STATE_SETTLED
for i in range(1, 589):
L.append("https://supreme.justia.com/cases/federal/us/" + str(i))
while (len(L) > 0):
print(len(L))
ctr = 0
curr_urls = []
while (ctr < 100 and len(L) > 0):
curr = L.popleft()
if not seen.__contains__(curr):
ctr += 1
seen.add(curr)
curr_urls.append(curr)
indices = [url for i, url in enumerate(curr_urls)]
rs = [grequests.get(u, timeout=3) for u in curr_urls]
grequests.map(rs)
for r in rs:
resp = r.response
if resp is None or resp.status_code is None or resp.status_code != 200:
print(f"REQUEUING {indices[i]}")
L.append(indices[i])
seen.remove(indices[i])
continue
resp_url = r.response.url
stripped_resp_url = strip_url(resp_url)
url_ct += 1
data = r.response.text
soup = BeautifulSoup(data, "lxml")
for link in soup.find_all('a'):
if link.has_attr('href'):
def async_request(urls, err_callback=None, pool_size=5, *args, **kw):
import grequests
tasks = [grequests.get(u, **kw) for u in urls]
return grequests.map(tasks, size=pool_size, exception_handler=err_callback)
def test_api_open_and_deposit_channel(
api_backend,
api_test_context,
api_raiden_service):
# let's create a new channel
first_partner_address = '0x61c808d82a3ac53231750dadc13c777b59310bd9'
token_address = '0xea674fdde714fd979de3edf0f56aa9716b898ec8'
settle_timeout = 1650
channel_data_obj = {
'partner_address': first_partner_address,
'token_address': token_address,
'settle_timeout': settle_timeout
}
request = grequests.put(
api_url_for(api_backend, 'channelsresource'),
json=channel_data_obj
)
response = request.send().response
assert_proper_response(response)
response = response.json()
expected_response = channel_data_obj
expected_response['balance'] = 0
expected_response['state'] = CHANNEL_STATE_OPENED
# can't know the channel address beforehand but make sure we get one
assert 'channel_address' in response
first_channel_address = response['channel_address']
expected_response['channel_address'] = response['channel_address']
assert response == expected_response
# now let's open a channel and make a deposit too
second_partner_address = '0x29fa6cf0cce24582a9b20db94be4b6e017896038'
balance = 100
channel_data_obj = {
'partner_address': second_partner_address,
'token_address': token_address,
'settle_timeout': settle_timeout,
'balance': balance
}
request = grequests.put(
api_url_for(api_backend, 'channelsresource'),
json=channel_data_obj
)
response = request.send().response
assert_proper_response(response)
response = response.json()
expected_response = channel_data_obj
expected_response['balance'] = balance
expected_response['state'] = CHANNEL_STATE_OPENED
# can't know the channel address beforehand but make sure we get one
assert 'channel_address' in response
expected_response['channel_address'] = response['channel_address']
second_channel_address = response['channel_address']
assert response == expected_response
# let's deposit on the first channel
request = grequests.patch(
api_url_for(
api_backend,
'channelsresourcebychanneladdress',
channel_address=first_channel_address
),
json={'balance': balance}
)
response = request.send().response
assert_proper_response(response)
response = response.json()
expected_response = {
'channel_address': first_channel_address,
'partner_address': first_partner_address,
'token_address': token_address,
'settle_timeout': settle_timeout,
'state': CHANNEL_STATE_OPENED,
'balance': balance
}
assert response == expected_response
# finally let's try querying for the second channel
request = grequests.get(
api_url_for(
api_backend,
'channelsresourcebychanneladdress',
channel_address=second_channel_address
)
)
response = request.send().response
assert_proper_response(response)
response = response.json()
expected_response = {
'channel_address': second_channel_address,
'partner_address': second_partner_address,
'token_address': token_address,
'settle_timeout': settle_timeout,
'state': CHANNEL_STATE_OPENED,
'balance': balance
}
assert response == expected_response
def run():
stocks = []
stock_map = {}
df = pd.read_csv(
'{data_path}/for_lives.csv'.format(data_path=env.data_path))
for i in range(len(df)):
ts_code = df.loc[i, 'ts_code']
if ts_code.startswith('688'):
continue
market = ts_code.split('.')[1].lower()
symbol = ts_code.split('.')[0]
name = df.loc[i, 'name']
stocks.append({'name': name, 'symbol': symbol, 'market': market})
stock_map['{market}{symbol}'.format(market=market, symbol=symbol)] = {
'local_max':
float(df.loc[i, 'COL_DAILY_LOCAL_MAX']),
'accumulation_aggressive':
float(df.loc[i, 'COL_ACCUMULATION_AGGRESSIVE'])
}
# for i in open('{data_path}/silent_ones'.format(data_path=env.data_path)):
# infos = i.strip().split(',')
# if infos[1].startswith('688'):
# continue
# stocks.append({
# 'name': infos[0],
# 'symbol': infos[1],
# 'market': infos[2]
# })
# stock_map['{market}{symbol}'.format(market=infos[2], symbol=infos[1])] = {
# 'local_max': float(infos[3])
# }
big_hands = {}
time_str = time.strftime("%Y-%m-%d", time.localtime())
pickle_file_name = '{buffer_path}/{time_str}@big_hand.pkl'.format(
buffer_path=env.buffer_path, time_str=time_str)
if os.path.exists(pickle_file_name):
with open(pickle_file_name, 'rb') as file:
try:
big_hands = pickle.load(file)
except EOFError:
big_hands = {}
batch_size = 500
req_list = []
for i in range(0, len(stocks), batch_size):
keys = []
for item in stocks[i:i + batch_size]:
query_key = '{market}{symbol}'.format(market=item['market'],
symbol=item['symbol'])
keys.append(query_key)
req_list.append(
grequests.get('http://hq.sinajs.cn/list={}'.format(
','.join(keys))))
while True:
time_a = time.time()
try:
responses = grequests.map(req_list)
print('====== {} ======'.format(
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
local_breaks = []
for response in responses:
res = response.text.strip().split(';\n')
for i in res:
j = i.split(',')
name = j[0].split('="')[1]
code = j[0].split('="')[0].split('_')[-1]
yesterday_closing_price = float(j[2])
current_price = float(j[3])
chg = (current_price / yesterday_closing_price - 1)
chg_display = '{}%'.format(round(chg * 100, 2))
local_max = stock_map[code]['local_max']
vols = int(j[8]) / 100
# update
if 'vols' in stock_map[code]:
last_vols = stock_map[code]['vols']
delta_vols = vols - last_vols
else:
# first time
stock_map[code]['vols'] = vols
delta_vols = 0
if delta_vols >= 10000:
if code not in big_hands:
big_hands[code] = {'max_delta_vols': delta_vols}
if delta_vols > big_hands[code]['max_delta_vols']:
big_hands[code]['max_delta_vols'] = delta_vols
with open(pickle_file_name, 'wb') as file:
pickle.dump(big_hands, file)
if current_price > local_max and 0.05 < chg < 0.098 and code in big_hands:
local_breaks.append({
'note':
'local break\t{code}\t{name}\tchg:{chg}\tprice:{price}\tbig_hands:{big_hands}\t\taccumulation:{accumulation}'
.format(
code=code,
name=name,
chg=chg_display,
price=round(current_price, 2),
big_hands=int(
big_hands[code]['max_delta_vols']),
accumulation=round(
stock_map[code]['accumulation_aggressive'],
2)),
'chg':
chg
})
#
# if 0.05 < chg < 0.098:
# if current_price > local_max:
# local_breaks.append({
# 'note': 'local break\t{code}\t{name}\tchg:{chg}\tprice:{price}'.format(code=code, name=name, chg=chg_display, price=round(current_price, 2)),
# 'chg': chg
# })
# else:
# # print('violent move\t{code}\t{name}\tchg:{chg}\tprice:{price}'.format(code=code, name=name, chg=chg_display, price=round(current_price, 2)))
# pass
local_breaks.sort(key=lambda x: x['chg'], reverse=True)
notes = [i['note'] for i in local_breaks]
print('\n'.join(notes))
except Exception as e:
print(e)
continue
time_b = time.time()
cost = time_b - time_a
time.sleep(1 - cost)
#!/usr/bin/python
# -*- coding: utf-8 -*-
#author:闻到 http://wendao123.cn
#抓取斗图啦http://www.doutula.com的表情包，并按文件夹存储
import requests
from bs4 import BeautifulSoup
import os, re, grequests
# import hashlib
url = 'http://www.doutula.com/article/list/?page={page}'
#请求队列，还未发出请求
urls = (grequests.get(url.format(page=page)) for page in range(1, 544))
#批量发出请求，得到响应的列表resps
response = grequests.map(urls)
for res in response:
doc = BeautifulSoup(res.text, 'lxml')
list_group_item = doc.find_all('a', class_='list-group-item random_list')
for group_item in list_group_item:
title = group_item.find(class_='random_title').text
# print(title[:-10])
img_urls = group_item.find_all(class_='lazy image_dtb img-responsive')
# print(img_urls)
for img_url in img_urls:
alt = img_url.get('alt')
true_img_url = img_url.get('data-backup')[:-4]
hz = true_img_url[-4:]
# print(alt,true_img_url,hz)
img_response = requests.get(true_img_url)
def test_payment_events_endpoints(api_backend, raiden_network,
token_addresses):
_, app1, app2 = raiden_network
amount = 200
identifier = 42
token_address = token_addresses[0]
target1_address = app1.raiden.address
target2_address = app2.raiden.address
api_server, _ = api_backend
# sending tokens to target 1
request = grequests.post(
api_url_for(
api_backend,
'token_target_paymentresource',
token_address=to_checksum_address(token_address),
target_address=to_checksum_address(target1_address),
),
json={
'amount': amount,
'identifier': identifier
},
)
request.send()
# sending some tokens to target 2
amount -= 10
request = grequests.post(
api_url_for(
api_backend,
'token_target_paymentresource',
token_address=to_checksum_address(token_address),
target_address=to_checksum_address(target2_address),
),
json={
'amount': amount,
'identifier': identifier
},
)
request.send()
# test endpoint without (partner and token)
request = grequests.get(api_url_for(
api_backend,
'paymentresource',
), )
response = request.send().response
assert_proper_response(response, HTTPStatus.OK)
response = response.json()
assert len(response) == 2
assert response[0]['event'] == 'EventPaymentSentSuccess'
assert response[1]['event'] == 'EventPaymentSentSuccess'
# test endpoint without partner
request = grequests.get(
api_url_for(
api_backend,
'token_paymentresource',
token_address=token_address,
), )
response = request.send().response
assert_proper_response(response, HTTPStatus.OK)
response = response.json()
assert len(response) == 2
assert response[0]['event'] == 'EventPaymentSentSuccess'
assert response[1]['event'] == 'EventPaymentSentSuccess'
# test endpoint for token and partner
request = grequests.get(
api_url_for(
api_backend,
'token_target_paymentresource',
token_address=token_address,
target_address=target1_address,
), )
response = request.send().response
assert_proper_response(response, HTTPStatus.OK)
response = response.json()
assert len(response) == 1
assert response[0]['event'] == 'EventPaymentSentSuccess'
"https://www.youtube.com/user/{}/about?hl=en".format(handle),
"https://twitter.com/{}".format(handle),
"https://www.instagram.com/{}".format(handle),
"https://www.facebook.com/{}".format(handle)
]
font = ImageFont.truetype("OpenSans-Italic.ttf", 30)
tcolor = (244, 210, 77)
text_pos1 = (50, 10)
text_pos2 = (620, 10)
text_pos3 = (50, 100)
text_pos4 = (620, 100)
img = Image.open("templates.jpg")
while True:
rs = (grequests.get(u) for u in link_list)
y, t, i, f = list(map(lambda r: r.text, grequests.map(rs)))
ys = fetch_youtube_subscriber(y)
tf = fetch_twitter_followers(t)
instagram_followers = fetch_instagram_followers(i)
ff = fetch_facebook_followers(f)
draw = ImageDraw.Draw(img)
draw.text(text_pos1, " {} \n YouTube".format(ys), fill=tcolor, font=font)
draw.text(text_pos2, " {} \n Twitter".format(tf), fill=tcolor, font=font)
draw.text(text_pos3,
" {} \n Instagram".format(instagram_followers),
fill=tcolor,
font=font)
draw.text(text_pos4,
" {} \n Facebook".format(ff),
def test_api_open_and_deposit_channel(
api_backend,
token_addresses,
reveal_timeout,
):
# let's create a new channel
first_partner_address = '0x61C808D82A3Ac53231750daDc13c777b59310bD9'
token_address = token_addresses[0]
settle_timeout = 1650
channel_data_obj = {
'partner_address': first_partner_address,
'token_address': to_checksum_address(token_address),
'settle_timeout': settle_timeout,
'reveal_timeout': reveal_timeout,
}
request = grequests.put(
api_url_for(
api_backend,
'channelsresource',
),
json=channel_data_obj,
)
response = request.send().response
assert_proper_response(response, HTTPStatus.CREATED)
response = response.json()
expected_response = channel_data_obj
expected_response['balance'] = 0
expected_response['state'] = CHANNEL_STATE_OPENED
expected_response[
'channel_identifier'] = assert_dicts_are_equal.IGNORE_VALUE
expected_response[
'token_network_identifier'] = assert_dicts_are_equal.IGNORE_VALUE
assert_dicts_are_equal(response, expected_response)
token_network_identifier = response['token_network_identifier']
# now let's open a channel and make a deposit too
second_partner_address = '0x29FA6cf0Cce24582a9B20DB94Be4B6E017896038'
balance = 100
channel_data_obj = {
'partner_address': second_partner_address,
'token_address': to_checksum_address(token_address),
'settle_timeout': settle_timeout,
'reveal_timeout': reveal_timeout,
'balance': balance,
}
request = grequests.put(
api_url_for(
api_backend,
'channelsresource',
),
json=channel_data_obj,
)
response = request.send().response
assert_proper_response(response, HTTPStatus.CREATED)
response = response.json()
expected_response = channel_data_obj
expected_response['balance'] = balance
expected_response['state'] = CHANNEL_STATE_OPENED
expected_response[
'channel_identifier'] = assert_dicts_are_equal.IGNORE_VALUE
expected_response['token_network_identifier'] = token_network_identifier
assert_dicts_are_equal(response, expected_response)
# assert depositing negative amount fails
request = grequests.patch(
api_url_for(
api_backend,
'channelsresourcebytokenandpartneraddress',
token_address=token_address,
partner_address=first_partner_address,
),
json={'total_deposit': -1000},
)
response = request.send().response
assert_proper_response(response, HTTPStatus.CONFLICT)
# let's deposit on the first channel
request = grequests.patch(
api_url_for(
api_backend,
'channelsresourcebytokenandpartneraddress',
token_address=token_address,
partner_address=first_partner_address,
),
json={'total_deposit': balance},
)
response = request.send().response
assert_proper_response(response)
response = response.json()
expected_response = {
'channel_identifier': assert_dicts_are_equal.IGNORE_VALUE,
'partner_address': first_partner_address,
'token_address': to_checksum_address(token_address),
'settle_timeout': settle_timeout,
'reveal_timeout': reveal_timeout,
'state': CHANNEL_STATE_OPENED,
'balance': balance,
'token_network_identifier': token_network_identifier,
}
assert_dicts_are_equal(response, expected_response)
# let's try querying for the second channel
request = grequests.get(
api_url_for(
api_backend,
'channelsresourcebytokenandpartneraddress',
token_address=token_address,
partner_address=second_partner_address,
), )
response = request.send().response
assert_proper_response(response)
response = response.json()
expected_response = {
'channel_identifier': assert_dicts_are_equal.IGNORE_VALUE,
'partner_address': second_partner_address,
'token_address': to_checksum_address(token_address),
'settle_timeout': settle_timeout,
'reveal_timeout': reveal_timeout,
'state': CHANNEL_STATE_OPENED,
'balance': balance,
'token_network_identifier': token_network_identifier,
}
assert_dicts_are_equal(response, expected_response)
# finally let's burn all eth and try to open another channel
api_server, _ = api_backend
burn_all_eth(api_server.rest_api.raiden_api.raiden)
channel_data_obj = {
'partner_address': '0xf3AF96F89b3d7CdcBE0C083690A28185Feb0b3CE',
'token_address': to_checksum_address(token_address),
'settle_timeout': settle_timeout,
'reveal_timeout': reveal_timeout,
'balance': 1,
}
request = grequests.put(
api_url_for(
api_backend,
'channelsresource',
),
json=channel_data_obj,
)
response = request.send().response
assert_proper_response(response, HTTPStatus.PAYMENT_REQUIRED)
response = response.json()
assert 'The account balance is below the estimated amount' in response[
'errors']
def muti_scrawl_page(urls):
rs = (grequests.get(u) for u in urls)
tmp = grequests.map(rs, size=10)
return tmp
def test_query_partners_by_token(api_backend, blockchain_services,
token_addresses):
first_partner_address = '0x61C808D82A3Ac53231750daDc13c777b59310bD9'
second_partner_address = '0x29FA6cf0Cce24582a9B20DB94Be4B6E017896038'
token_address = token_addresses[0]
settle_timeout = 1650
channel_data_obj = {
'partner_address': first_partner_address,
'token_address': to_checksum_address(token_address),
'settle_timeout': settle_timeout,
}
request = grequests.put(
api_url_for(
api_backend,
'channelsresource',
),
json=channel_data_obj,
)
response = request.send().response
assert_proper_response(response, HTTPStatus.CREATED)
response = response.json()
channel_data_obj['partner_address'] = second_partner_address
request = grequests.put(
api_url_for(
api_backend,
'channelsresource',
),
json=channel_data_obj,
)
response = request.send().response
assert_proper_response(response, HTTPStatus.CREATED)
response = response.json()
# and a channel for another token
channel_data_obj[
'partner_address'] = '0xb07937AbA15304FBBB0Bf6454a9377a76E3dD39E'
channel_data_obj['token_address'] = to_checksum_address(token_address)
request = grequests.put(
api_url_for(
api_backend,
'channelsresource',
),
json=channel_data_obj,
)
response = request.send().response
assert_proper_response(response, HTTPStatus.CREATED)
# and now let's query our partners per token for the first token
request = grequests.get(
api_url_for(
api_backend,
'partnersresourcebytokenaddress',
token_address=to_checksum_address(token_address),
), )
response = request.send().response
assert_proper_response(response)
response = response.json()
expected_response = [
{
'partner_address':
first_partner_address,
'channel':
'/api/1/channels/{}/{}'.format(
to_checksum_address(token_address),
to_checksum_address(first_partner_address),
),
},
{
'partner_address':
second_partner_address,
'channel':
'/api/1/channels/{}/{}'.format(
to_checksum_address(token_address),
to_checksum_address(second_partner_address),
),
},
]
assert all(r in response for r in expected_response)
from itertools import permutations
import grequests
s = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
s = list(s)
base = 'https://pokemongo.gamepress.gg/sites/pokemongo/files/styles/240w/public/flork-images/'
for i in range(1, 5):
print(i)
rs = (grequests.get(f'{base}{"".join([str(y) for y in x])}.png')
for x in permutations(s, i))
requests = grequests.map(rs)
for response in requests:
if response:
print(response.url)
urls = [
'http://www.heroku.com', 'http://python-tablib.org', 'http://httpbin.org',
'http://python-requests.org', 'http://kennethreitz.com'
]
rs = (grequests.get(u) for u in urls)
requests = grequests.map(rs)
for response in requests:
print(response)
def test_get_connection_managers_info(api_backend, token_addresses):
# check that there are no registered tokens
request = grequests.get(
api_url_for(api_backend, 'connectionsinforesource'), )
response = request.send().response
result = response.json()
assert len(result) == 0
funds = 100
token_address1 = to_checksum_address(token_addresses[0])
connect_data_obj = {
'funds': funds,
}
request = grequests.put(
api_url_for(
api_backend,
'connectionsresource',
token_address=token_address1,
),
json=connect_data_obj,
)
response = request.send().response
assert_no_content_response(response)
# check that there now is one registered channel manager
request = grequests.get(
api_url_for(api_backend, 'connectionsinforesource'), )
response = request.send().response
result = response.json()
assert isinstance(result, dict) and len(result.keys()) == 1
assert token_address1 in result
assert isinstance(result[token_address1], dict)
assert set(result[token_address1].keys()) == {
'funds', 'sum_deposits', 'channels'
}
funds = 100
token_address2 = to_checksum_address(token_addresses[1])
connect_data_obj = {
'funds': funds,
}
request = grequests.put(
api_url_for(
api_backend,
'connectionsresource',
token_address=token_address2,
),
json=connect_data_obj,
)
response = request.send().response
assert_no_content_response(response)
# check that there now are two registered channel managers
request = grequests.get(
api_url_for(api_backend, 'connectionsinforesource'), )
response = request.send().response
result = response.json()
assert isinstance(result, dict) and len(result.keys()) == 2
assert token_address2 in result
assert isinstance(result[token_address2], dict)
assert set(result[token_address2].keys()) == {
'funds', 'sum_deposits', 'channels'
}
# first page
visited = set('qds.html')
stack = [requests.get(root + 'qds.html').text]
phrases = []
while len(stack) != 0:
html = stack.pop()
soup = BeautifulSoup(html, 'html.parser')
# get phrase
if soup.font:
phrase = soup.font.contents[1]
phrases.append(phrase)
print(phrase)
# get more links
links = []
for link in soup.find_all('a'):
link = link.get('href')
if link in visited:
continue
visited.add(link)
links.append(grequests.get(root + link))
print(link)
# add all links responses to stack
[stack.append(link.text) for link in grequests.map(links)]
[print(phrase) for phrase in sorted(phrases)]
def main():
# globals
global concurrent # how many concurrent requests to make at once
global host # collection of all Page objects (see class Page), starting w/ passed target URL
global dictionary # loaded words
global quiet # shh
global schema # webpage request schema (http:// or https://)
# collect options and arguments
try:
options, a = getopt.getopt(sys.argv[1:], "c:halto:q", [
"conc=",
"help",
"all",
"listable",
"tree",
"out=",
"quiet"
])
except getopt.GetoptError as err:
print(str(err))
usage()
sys.exit(1)
# init flag / config variables
man_args = a # program takes mandatory positional args, so anything not option-flagged goes here.
all_flag = False
list_flag = False
tree_flag = False
out_file = None
concurrent = 15
quiet = False
# now handle our options
for option, argument in options:
if option in ("-c", "--conc"):
try:
concurrent = int(argument)
except ValueError:
print("We both know %s needs to be passed a number. Don't be a s******d.\n" % option)
usage()
sys.exit(1)
elif option in ("-h", "--help"):
usage()
sys.exit(0)
elif option in ("-a", "--all"):
all_flag = True
elif option in ("-t", "--tree"):
tree_flag = True
elif option in ("-l", "--listable"):
list_flag = True
elif option in ("-o", "--out"):
out_file = argument
elif option in ("-q", "--quiet"):
quiet = True
else:
assert False, "unhandled option"
# handle mandatory / positional arguments
if len(man_args) < 2:
print("Error: Missing %d of 2 mandatory arguments!" % (2 - len(man_args)))
print(" required syntax: slasher.py <host> <dictionary>")
print(" run with -h or --help for usage")
sys.exit(1)
elif len(man_args) >= 3:
print("[!] Unrecognized positional arguments:")
print(" " + str(man_args[2:]) + " will not be used!")
while True:
r = input(" Continue? Y/N\n > ")
if r in "yY":
print("[*] Proceeding, gnoring unrecognized args...")
break
elif r in "nN":
print("[!] User chose to end scan.")
sys.exit(1)
print("\n[!] Invalid response. Please try again.")
# setup our starting page and get path to our wordlist
host = [ [Page(man_args[0], None, False, True)] ] # 2d list. ctrl+f "main scan loop" for deets on why it's this way
top = host[0][0] # this is just for convenience when we need to reference only the main site page.
list_path = man_args[1]
# get schema
if "https://" in top.uri:
schema = "https://"
top.uri = top.uri[8:]
elif "http://" in top.uri:
schema = "http://"
top.uri = top.uri[7:]
else:
schema = "http://"
# take credit ;)
if 1:
print("")
print(" # ##")
print(" ## ###")
print(" ## ### ###")
print(" ###########")
print(" #############")
print(" ################")
print(" ##################")
print(" ####### ############")
print(" ###### ############")
print(" ### ############")
print("\n----------------------------")
print(" slasher.py v1.1.0")
print(" by k-ddev")
print("")
print(" A directory scanner\n")
else:
print("\n----------------------------")
print(" Target host - %s" % top.uri)
print(" Wordlist - %s" % list_path)
print(" Requests - %d" % concurrent)
print(" Enabled modes:")
if all_flag:
print(" [*] scan all")
if list_flag:
print(" > scan listable")
print("----------------------------\n")
# make sure top is even connectable
print ("[*] Checking for remote host connection at [%s]" % (schema + top.uri))
response = grequests.map([grequests.get(schema + top.uri)])[0]
try:
if response.status_code >= 200 and response.status_code < 300:
print ("[*] Great Success!")
else:
print ("[!] No connection to [%s]\n[*] Exiting." % (schema + top.uri))
sys.exit(1)
except:
print ("[!] No connection to [%s]\n[*] Exiting." % (schema + top.uri))
sys.exit(1)
# make sure the dictionary works
print("\n[*] Checking dictionary: [%s]" % list_path)
try:
with open(list_path) as f:
dictionary = f.read().strip().split('\n')
print("[*] Successfully set up them the bomb with [%d] paths to try" % (len(dictionary)))
except IOError:
print("[!] Something went wrong :(")
print("[*] Failed to read [%s]" % list_path)
sys.exit(1)
# main scan loop
"""
Here's where our collection 'host' comes into play. Each index of host
represents a level of the site. host[0] as seen already contains only
the top page. host[1] will be each page below top, and host[2] will
contain every subpage of every page in host[1], and so on. Our loop
will scan every page in host[i], collecting found pages into host[i+1].
If host[i+1] contains no pages, we know there's nothing else to find.
"""
print("\n[*] Starting scan.")
depth = 0
while True:
# allocate a new index of host for any pages we find
host.append([])
try:
assert len(host) == depth + 2
except AssertionError:
print("[!] Something weird happened. The greatest depth of the site as we know it")
print(" should be 1 greater than the depth of the pages currently being scanned,")
print(" but is not.")
sys.exit(0)
# scan every page on current depth and collect found pages
for page in host[depth]:
if all_flag or page.may_be_directory:
if not list_flag and page.is_listable:
print("\r\033[K\n[*] skipped listable directory: %s" % page.get_url())
print(" specify -l to scan listable")
else:
found = scan(page)
for f in found:
host[-1].append(f)
# if no more pages were found, we're done. Otherwise, go to next depth
if len(host[-1]) == 0:
break
else:
depth += 1
print("\r\033[K\n[*] All their subpage are belong to us!")
def run():
symbol2code = {}
stock_map = {}
for stock in main_session.query(models.DailyBasic).all():
ts_code = stock.ts_code
market = ts_code.split('.')[1].lower()
symbol = ts_code.split('.')[0]
code = '{market}{symbol}'.format(market=market, symbol=symbol)
symbol2code[symbol] = code
stock_map[code] = {'circ_mv': float(stock.circ_mv)}
batch_size = 500
req_list = []
for i in range(0, len(target_symbols), batch_size):
keys = []
for symbol in target_symbols[i:i + batch_size]:
query_key = symbol2code[symbol]
keys.append(query_key)
req_list.append(
grequests.get('http://hq.sinajs.cn/list={}'.format(
','.join(keys))))
while True:
time_a = time.time()
try:
responses = grequests.map(req_list)
print('====== {} ======'.format(
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
displays = []
for response in responses:
res = response.text.strip().split(';\n')
for i in res:
j = i.split(',')
name = j[0].split('="')[1]
code = j[0].split('="')[0].split('_')[-1]
yesterday_closing_price = float(j[2])
current_price = float(j[3])
today_max_price = float(j[4])
buy_one_price = float(j[6])
buy_one_vol = float(j[10])
today_limit_price = round(yesterday_closing_price * 1.2, 2)
chg = (current_price / yesterday_closing_price - 1)
chg_display = '{}%'.format(round(chg * 100, 2))
circ_mv = stock_map[code]['circ_mv']
if_display = False
type = 1
if today_max_price == today_limit_price: #摸过板的
if_display = True
if buy_one_price < today_limit_price: #开板
if_display = True
elif buy_one_price * buy_one_vol < 10000000: #封单小于1kw
if_display = True
type = 2
elif chg > 0.1:
if_display = True
# if_display = True
if if_display:
if type == 2:
displays.append({
'note':
'{code}\t{name}\tchg:{chg}\tprice:{price}\tcirc_mv:{circ_mv}亿\t封单:{vol}手'
.format(code=code,
name=name,
chg=chg_display,
price=round(current_price, 2),
circ_mv=int(circ_mv),
vol=int(buy_one_vol / 100)),
'chg':
chg
})
else:
displays.append({
'note':
'{code}\t{name}\tchg:{chg}\tprice:{price}\tcirc_mv:{circ_mv}亿'
.format(code=code,
name=name,
chg=chg_display,
price=round(current_price, 2),
circ_mv=int(circ_mv)),
'chg':
chg
})
displays.sort(key=lambda x: x['chg'], reverse=False)
notes = [i['note'] for i in displays]
print('\n'.join(notes))
except Exception as e:
print(e)
continue
time_b = time.time()
cost = time_b - time_a
time.sleep(1 - cost)
def test_query_blockchain_events(
api_backend,
api_test_context,
api_raiden_service):
# Adding some mock events. Some of these events should not normally contain
# a block number but for the purposes of making sure block numbers propagate
# in the API logic I am adding them here and testing for them later.
api_test_context.add_events([{
'_event_type': 'TokenAdded',
'block_number': 1,
'channel_manager_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9',
'token_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9'
}, {
'_event_type': 'TokenAdded',
'block_number': 13,
'channel_manager_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9',
'token_address': '0xea674fdde714fd979de3edf0f56aa9716b898ec8'
}, {
'_event_type': 'ChannelNew',
'settle_timeout': 10,
'netting_channel': '0xea674fdde714fd979de3edf0f56aa9716b898ec8',
'participant1': '0x4894a542053248e0c504e3def2048c08f73e1ca6',
'participant2': '0x356857Cd22CBEFccDa4e96AF13b408623473237A',
'block_number': 15,
}, {
'_event_type': 'ChannelNew',
'settle_timeout': 10,
'netting_channel': '0xa193fb0032c8635d590f8f31be9f70bd12451b1e',
'participant1': '0xcd111aa492a9c77a367c36e6d6af8e6f212e0c8e',
'participant2': '0x88bacc4ddc8f8a5987e1b990bb7f9e8430b24f1a',
'block_number': 100,
}, {
'_event_type': 'ChannelNewBalance',
'token_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9',
'participant': '0xcd111aa492a9c77a367c36e6d6af8e6f212e0c8e',
'balance': 200,
'block_number': 20,
}, {
'_event_type': 'ChannelNewBalance',
'token_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9',
'participant': '0x00472c1e4275230354dbe5007a5976053f12610a',
'balance': 650,
'block_number': 150,
}, {
'_event_type': 'ChannelSettled',
'block_number': 35,
}, {
'_event_type': 'ChannelSettled',
'block_number': 250,
}])
# and now let's query the network events for 'TokenAdded' for blocks 1-10
request = grequests.get(
api_url_for(
api_backend,
'networkeventsresource',
from_block=0,
to_block=10
)
)
response = request.send().response
assert_proper_response(response)
response = json.loads(response._content)
assert len(response) == 1
assert response[0] == {
'event_type': 'TokenAdded',
'block_number': 1,
'channel_manager_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9',
'token_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9'
}
# query ChannelNew event for a token
api_test_context.specify_token_for_channelnew('0x61c808d82a3ac53231750dadc13c777b59310bd9')
request = grequests.get(
api_url_for(
api_backend,
'tokeneventsresource',
token_address='0x61c808d82a3ac53231750dadc13c777b59310bd9',
from_block=5,
to_block=20
)
)
response = request.send().response
assert_proper_response(response)
response = json.loads(response._content)
assert len(response) == 1
assert response[0] == {
'event_type': 'ChannelNew',
'settle_timeout': 10,
'netting_channel': '0xea674fdde714fd979de3edf0f56aa9716b898ec8',
'participant1': '0x4894a542053248e0c504e3def2048c08f73e1ca6',
'participant2': '0x356857Cd22CBEFccDa4e96AF13b408623473237A',
'block_number': 15,
}
# finally query for some channel related events
# Note: No need to test em all since this does not test the implementation
# of `get_channel_events()` but just makes sure the proper data make it there
api_test_context.specify_channel_for_events('0xedbaf3c5100302dcdda53269322f3730b1f0416d')
request = grequests.get(
api_url_for(
api_backend,
'channeleventsresource',
channel_address='0xedbaf3c5100302dcdda53269322f3730b1f0416d',
from_block=10,
to_block=90
)
)
response = request.send().response
assert_proper_response(response)
response = json.loads(response._content)
assert len(response) == 2
assert response[0] == {
'event_type': 'ChannelNewBalance',
'token_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9',
'participant': '0xcd111aa492a9c77a367c36e6d6af8e6f212e0c8e',
'balance': 200,
'block_number': 20,
}
assert response[1] == {
'event_type': 'ChannelSettled',
'block_number': 35,
}
