Example #1
0
def list_members(request, microcosm_id):
    offset = int(request.GET.get('offset', 0))

    microcosm_url, params, headers = Microcosm.build_request(request.get_host(), id=microcosm_id,
        offset=offset, access_token=request.access_token)
    request.view_requests.append(grequests.get(microcosm_url, params=params, headers=headers))
    try:
        responses = response_list_to_dict(grequests.map(request.view_requests))
    except APIException as exc:
        return respond_with_error(request, exc)
    microcosm = Microcosm.from_api_response(responses[microcosm_url])

    roles_url, params, headers = RoleList.build_request(request.META['HTTP_HOST'], id=microcosm_id,
        offset=offset, access_token=request.access_token)
    request.view_requests.append(grequests.get(roles_url, params=params, headers=headers))
    try:
        responses = response_list_to_dict(grequests.map(request.view_requests))
    except APIException as exc:
        return respond_with_error(request, exc)
    roles = RoleList.from_api_response(responses[roles_url])

    view_data = {
        'user': Profile(responses[request.whoami_url], summary=False) if request.whoami_url else None,
        'site': Site(responses[request.site_url]),
        'site_section': 'memberships',
        'content': microcosm,
        'memberships': roles,
        'item_type': 'microcosm',
        'pagination': build_pagination_links(responses[roles_url]['roles']['links'], roles.items)
    }

    return render(request, members_list_template, view_data)
Example #2
0
    def process_request(self, request):
        """
        Checks for access_token cookie and appends it to the request object if present.

        All request objects have a view_requests attribute which is a list of requests
        that will be executed by grequests to fetch data for the view.
        """

        request.access_token = None
        request.whoami_url = ''
        request.view_requests = []

        if request.COOKIES.has_key('access_token'):
            # Clean up empty access token.
            if request.COOKIES['access_token'] == '':
                response = HttpResponseRedirect('/')
                response.set_cookie('access_token', '', expires="Thu, 01 Jan 1970 00:00:00 GMT")
                return response
            request.access_token = request.COOKIES['access_token']
            request.whoami_url, params, headers = WhoAmI.build_request(request.get_host(), request.access_token)
            request.view_requests.append(grequests.get(request.whoami_url, params=params, headers=headers))
            newrelic.agent.add_custom_parameter('access_token', request.access_token[:6])

        request.site_url, params, headers = Site.build_request(request.get_host())
        request.view_requests.append(grequests.get(request.site_url, params=params, headers=headers))
Example #3
0
def geturls(urls,headers=None):
    movies      = []
    movie_hrefs = []

    # 使用grequest进行异步获取网页
    movies_requests = [grequests.get(u,headers=headers) for u in urls]
    movies_datas    = grequests.map(movies_requests)

    for movies_d in movies_datas:
        movies_soup = BeautifulSoup(movies_d.text,'lxml')
        names       = movies_soup.select('div.item > div.info > div.hd > a > span:nth-of-type(1)') 
        ranks       = movies_soup.select('div.item > div.pic > em')
        hrefs       = movies_soup.select('div.item > div.info > div.hd > a')
        for name,rank,href in zip(names,ranks,hrefs):
            movie   = MovieClass.Movie(name.get_text(),rank.get_text(),href.get('href'))
            movies.append(movie)
        for href in hrefs:
            movie_hrefs.append(href.get('href'))
    # 异步获取250个网页
    movie_requests  = [grequests.get(u,headers=headers) for u in movie_hrefs]
    movie_webs     = grequests.map(movie_requests)
    # 将网页数据和电影初步数据保存下来
    with open('moviehtmls.txt','wb') as fs:
        pickle.dump(movie_webs,fs)
    with open('moviesdata.txt','wb') as fs:
        pickle.dump(movies,fs)

    return movies,movie_webs
Example #4
0
  def handleCommand(self, update):
    self.bot.sendChatAction(chat_id=update.message.chat_id, action=telegram.ChatAction.TYPING)
    apiServer="localhost"
    hacklabRequests = [
        grequests.get(url="http://{}/pi_api/temp/".format(apiServer), params={"a":"getTemp"}),              # 0: temperature
        grequests.get(url="http://{}/pi_api/humidity/".format(apiServer), params={"a":"getHumidity"}),      # 1: humidity
        grequests.get(url="http://{}/pi_api/gpio/".format(apiServer), params={"a":"readPin", "pin":"1"}),   # 2: electronics
        grequests.get(url="http://{}/pi_api/gpio/".format(apiServer), params={"a":"readPin", "pin":"0"}),   # 3: mechanics
        grequests.get(url="http://{}/pi_api/pir/".format(apiServer), params={"a":"getStatus"})]             # 4: PIR
    hacklabResponses = grequests.map(hacklabRequests)
    if None in hacklabResponses:
        raise ConnectionError("A request for the /hacklab command failed!")

    temperature = json.loads(hacklabResponses[0].text)['data']
    humidity = json.loads(hacklabResponses[1].text)['data']
    electronicsLight = json.loads(hacklabResponses[2].text)['data'] == "0"
    mechanicsLight = json.loads(hacklabResponses[3].text)['data'] == "0"
    pirStatus = json.loads(hacklabResponses[4].text)['time']

    if not electronicsLight and not mechanicsLight:
      lightStatus = "Lights are off. Hacklab is probably empty."
    elif electronicsLight != mechanicsLight:
      lightStatus = "Lights are on in the {} room.".format(("electronics" if electronicsLight else "mechanics"))
    else:
      lightStatus = "Lights are on in both rooms!"

    responseMessage = "{} Last movement at {}. Temperature is {:.1f}\u00b0C. Humidity is {}%".format(lightStatus, pirStatus, temperature, humidity)
    self.bot.sendMessage(chat_id=update.message.chat_id, text=responseMessage)
Example #5
0
    def server_error(request, exception=None):
        view_data = {}
        view_requests = []

        if request.COOKIES.has_key('access_token'):
            request.access_token = request.COOKIES['access_token']
            whoami_url, params, headers = WhoAmI.build_request(request.get_host(), request.access_token)
            view_requests.append(grequests.get(whoami_url, params=params, headers=headers))

        site_url, params, headers = Site.build_request(request.get_host())
        view_requests.append(grequests.get(request.site_url, params=params, headers=headers))

        responses = response_list_to_dict(grequests.map(view_requests))
        if request.whoami_url:
            profile = Profile(responses[whoami_url], summary=False)
            view_data['user'] = profile
            newrelic.agent.add_custom_parameter('profile_name', profile.profile_name)
            newrelic.agent.add_custom_parameter('profile_id', profile.id)
            newrelic.agent.add_custom_parameter('user_id', profile.user_id)

        site = Site(responses[site_url])
        view_data['site'] = site
        newrelic.agent.add_custom_parameter('site', site.subdomain_key)

        # Provide detailed error if returned in the response.
        if hasattr(exception, 'detail'):
            if exception.detail.has_key('errorDetail'):
                view_data['detail'] = exception.detail['errorDetail']

        context = RequestContext(request, view_data)
        return HttpResponseServerError(loader.get_template('500.html').render(context))
Example #6
0
def reap(file_name):
    config = ConfigParser.ConfigParser()
    config.read('combine.cfg')

    inbound_url_file = config.get('Reaper', 'inbound_urls')
    outbound_url_file = config.get('Reaper', 'outbound_urls')

    with open(inbound_url_file, 'rb') as f:
        inbound_urls = [url.rstrip('\n') for url in f.readlines()]
    with open(outbound_url_file, 'rb') as f:
        outbound_urls = [url.rstrip('\n') for url in f.readlines()]
    headers = {'User-Agent': 'harvest.py'}

    sys.stderr.write('Fetching inbound URLs\n')
    reqs = [grequests.get(url, headers=headers) for url in inbound_urls]
    inbound_responses = grequests.map(reqs)
    inbound_harvest = [(response.url, response.status_code, response.text) for response in inbound_responses]

    sys.stderr.write('Fetching outbound URLs\n')
    reqs = [grequests.get(url, headers=headers) for url in outbound_urls]
    outbound_responses = grequests.map(reqs)
    outbound_harvest = [(response.url, response.status_code, response.text) for response in outbound_responses]
    
    sys.stderr.write('Storing raw feeds in %s\n' % file_name)
    harvest = {'inbound': inbound_harvest, 'outbound': outbound_harvest}

    with open(file_name, 'wb') as f:
        json.dump(harvest, f, indent=2)
Example #7
0
def get_pages(response, topics, link_list):
    i = 0
    nb_link = 0
    nb_requests = 0
    for res in response:
        # get each page from the last
        list_pages = []
        while int(link_list[i].split('-')[3]) > 0:
            previous = link_list[i].split('-')
            previous_page = int(previous[3])
            previous_page -= 1
            previous[3] = str(previous_page)
            link_list[i] = '-'.join(previous)
            list_pages.append(link_list[i])
            print link_list[i]
            if nb_requests > 8:
                rs = (grequests.get(u) for u in list_pages)
                laresponse = grequests.map(rs)
                get_pseudos(laresponse)
                nb_requests = 0
                del list_pages[:]
            nb_requests+=1
            
        rs = (grequests.get(u, stream=True) for u in list_pages)
        laresponse = grequests.map(rs)
        get_pseudos(laresponse)
        res.close()
        i+=1
    return
Example #8
0
def parse(url):
    page = requests.get(url)
    soup = bs(page.text, 'lxml')
    if url == start_url:
        async_list = []
        for i in xrange(1, 6):
            print (url+'?&pg={}'.format(i))
            rs = (grequests.get(url+'?&pg={}'.format(i), hooks = {'response' : scrape}))
            async_list.append(rs)
        grequests.map(async_list)


    active_sel = soup.find('span', 'zg_selected').find_next()
    if active_sel.name == 'ul':
        links_list = active_sel.find_all('li')
        for link_list in links_list:
            link = link_list.find('a')['href'].encode('utf-8')
            async_list = []
            for i in xrange(1, 6):
                print (link+'?&pg={}'.format(i))
                rs = (grequests.get(link+'?&pg={}'.format(i), hooks = {'response' : scrape}))
                async_list.append(rs)

            grequests.map(async_list)
            parse(link)
Example #9
0
def single(request, profile_id):
    """
    Display a single profile by ID.
    """

    # Fetch profile details.
    profile_url, params, headers = Profile.build_request(request.get_host(), profile_id, access_token=request.access_token)
    request.view_requests.append(grequests.get(profile_url, params=params, headers=headers))

    # Fetch items created by this profile.
    search_q = 'type:conversation type:event type:huddle type:comment authorId:%s' % profile_id
    search_params = {'limit': 10, 'q': search_q, 'sort': 'date'}
    search_url, params, headers = Search.build_request(request.get_host(), search_params,
        access_token=request.access_token)
    request.view_requests.append(grequests.get(search_url, params=params, headers=headers))

    try:
        responses = response_list_to_dict(grequests.map(request.view_requests))
    except APIException as exc:
        return respond_with_error(request, exc)

    user = Profile(responses[request.whoami_url], summary=False) if request.whoami_url else None
    profile = Profile(responses[profile_url], summary=False)

    view_data = {
        'user': user,
        'content': profile,
        'item_type': 'profile',
        'site': Site(responses[request.site_url]),
        'search': Search.from_api_response(responses[search_url]),
        'site_section': 'people'
    }
    return render(request, single_template, view_data)
Example #10
0
def main():
    resource.setrlimit(resource.RLIMIT_NOFILE, (2048, 2048))
    hashes = set()
    past_urls = set()

    args = setup_args(sys.argv[1:])
    if args.config:
        cfg = config(args, args.config)
    else:
        cfg = config(args, 'maltrieve.cfg')
    cfg.check_proxy()

    hashes = load_hashes('hashes.json')
    past_urls = load_urls('urls.json')

    print "Processing source URLs"

    # TODO: Replace with plugins
    source_urls = {'https://zeustracker.abuse.ch/monitor.php?urlfeed=binaries': process_xml_list_desc,
                   'http://www.malwaredomainlist.com/hostslist/mdl.xml': process_xml_list_desc,
                   'http://malc0de.com/rss/': process_xml_list_desc,
                   'http://vxvault.net/URL_List.php': process_simple_list,
                   'http://urlquery.net/': process_urlquery,
                   'http://support.clean-mx.de/clean-mx/rss?scope=viruses&limit=0%2C64': process_xml_list_title,
                   'http://malwareurls.joxeankoret.com/normal.txt': process_simple_list}
    headers = {'User-Agent': 'Maltrieve'}

    reqs = [grequests.get(url, timeout=60, headers=headers, proxies=cfg.proxy) for url in source_urls]
    source_lists = grequests.map(reqs)

    print "Completed source processing"

    headers['User-Agent'] = cfg.useragent
    malware_urls = set()
    for response in source_lists:
        if hasattr(response, 'status_code') and response.status_code == 200:
            malware_urls.update(source_urls[response.url](response.text))

    if cfg.inputfile:
        with open(cfg.inputfile, 'rb') as f:
            moar_urls = list(f)
        malware_urls.update(moar_urls)

    print "Downloading samples, check log for details"

    malware_urls -= past_urls
    reqs = [grequests.get(url, timeout=60, headers=headers, proxies=cfg.proxy) for url in malware_urls]
    for chunk in chunker(reqs, 32):
        malware_downloads = grequests.map(chunk)
        for each in malware_downloads:
            if not each or each.status_code != 200:
                continue
            if save_malware(each, cfg):
                past_urls.add(each.url)

    print "Completed downloads"

    save_urls(past_urls, 'urls.json')
    save_hashes(hashes, 'hashes.json')
    sort_downloads()
Example #11
0
    def forbidden(request):
        view_data = {}
        view_requests = []

        if request.COOKIES.has_key('access_token'):
            request.access_token = request.COOKIES['access_token']
            whoami_url, params, headers = WhoAmI.build_request(request.get_host(), request.access_token)
            view_requests.append(grequests.get(whoami_url, params=params, headers=headers))

        site_url, params, headers = Site.build_request(request.get_host())
        view_requests.append(grequests.get(request.site_url, params=params, headers=headers))

        responses = response_list_to_dict(grequests.map(view_requests))
        if request.whoami_url:
            profile = Profile(responses[whoami_url], summary=False)
            view_data['user'] = profile
            newrelic.agent.add_custom_parameter('profile_name', profile.profile_name)
            newrelic.agent.add_custom_parameter('profile_id', profile.id)
            newrelic.agent.add_custom_parameter('user_id', profile.user_id)

        site = Site(responses[site_url])
        view_data['site'] = site
        newrelic.agent.add_custom_parameter('site', site.subdomain_key)

        context = RequestContext(request, view_data)
        return HttpResponseForbidden(loader.get_template('403.html').render(context))
Example #12
0
def parse(url):
    if url in start_urls:
        async_list = []
        print (url)
        for i in xrange(1, 6):
            rs = (grequests.get(url+'?&pg={}'.format(i), hooks = {'response' : scrape}))
            async_list.append(rs)
        grequests.map(async_list)
    page = requests.get(url)
    soup = bs(page.text, 'lxml')
    try:
        active_sel = soup.find('span', 'zg_selected').find_next()
        if active_sel.name == 'ul':
            links_lists= active_sel.find_all('li')
            asins = pool.map(multiparse, links_lists)
            for asin in asins:
                async_list = []
                print (asin)
                for i in xrange(1, 6):
                    rs = (grequests.get(asin+'?&pg={}'.format(i), hooks = {'response' : scrape}))
                    async_list.append(rs)
                parse(asin)
                grequests.map(async_list)


    except:
        parse(url)
def parse(url):
    if url in start_urls:
        async_list = []
        print(url)
        rs = grequests.get(url, headers=headers, hooks={"response": scrape})
        async_list.append(rs)
        grequests.map(async_list)
    page = requests.get(url)
    soup = bs(page.text, "lxml")
    try:
        active_sel = soup.find("span", "zg_selected").find_next()
        if active_sel.name == "ul":
            links_lists = active_sel.find_all("li")
            asins = pool.map(multiparse, links_lists)
            for asin in asins:
                async_list = []
                print(asin)
                for i in xrange(1, 6):
                    rs = grequests.get(asin + "?&pg={}".format(i), hooks={"response": scrape})
                    async_list.append(rs)
                parse(asin)
                grequests.map(async_list)

    except:
        parse(url)
Example #14
0
def get_route_info_from_infotraffic(known_lines_csv: str, known_stations_csv: str)-> Dict[int, Tuple[Route, Route]]:
	root = 'http://86.122.170.105:61978/html/timpi/'
	urls = [grequests.get(root + 'tram.php', stream=False),
	        grequests.get(root + 'trol.php', stream=False),
	        grequests.get(root + 'auto.php', stream=False)]

	known_lines = { line.line_id: line for line in importer.parse_lines_from_csv(known_lines_csv) }
	known_lines = known_lines  # type: Dict[int, Line]
	known_stations = { station.raw_name: station for station in importer.parse_stations_from_csv(known_stations_csv) }
	known_stations = known_stations  # type: Dict[str, Station]
	line_id_re = re.compile("param1=(\d+)")
	line_id_to_routes = {}  # type: Dict[int, Tuple[Route, Route]]
	for page in grequests.imap(urls, size=len(urls), exception_handler=exception_handler):
		page.raise_for_status()
		if page.status_code == requests.codes.ok:
			soup = bs4.BeautifulSoup(page.text, "html.parser")
			unknown_lines = { }  # type: Dict[int, str]
			line_requests = []
			for a in soup.select("div p a"):
				line_id = int(line_id_re.search(a['href']).group(1))
				line = known_lines.get(line_id, None)
				if not line:
					line_name = a['title'] if a.has_attr('title') else None
					if line_name is None:
						img = a.select("img")[0]
						line_name = img['alt'] if img and img.has_attr('alt') else 'unknown'
					unknown_lines[line_id] = line_name
					print("WARNING: unknown line '{line_name}' (line ID: {line_id}) encountered at {url}"
					      .format(line_name=line_name, line_id=line_id, url=page.url))
				line_requests.append(grequests.get(root + a['href'], stream=False))

			for line_response in grequests.imap(line_requests, size=6, exception_handler=exception_handler):
				line_id = int(line_id_re.search(line_response.url).group(1))
				routes = parse_arrivals_from_infotrafic(line_id, known_stations, line_response, include_unknown_stations=True)
				line = known_lines.get(line_id, None)
				line_name = line.line_name if line is not None else unknown_lines.get(line_id, "unknown")
				route1 = route2 = None
				for route_id, route in enumerate(routes):
					valid_stations = []
					for station, arrival in route:
						if not isinstance(station, Station):
							print("WARNING: unknown station '{raw_station_name}' encountered in route {route_id} of line {line_name} (line ID: {line_id})"
							      .format(line_name=line_name, line_id=line_id, route_id=route_id, raw_station_name=station))
						else:
							if not station.lng or not station.lat:
								print("WARNING: station '{station_name}' (station ID: {station_id}) has no GPS coordinates defined"
								      .format(station_name=station.friendly_name, station_id=station.station_id))
							valid_stations.append(station)

					if valid_stations and line is not None:
						if route_id == 0:
							route1 = Route(route_id, line.route_name_1, line.line_id, valid_stations)
						elif route_id == 1:
							route2 = Route(route_id, line.route_name_2, line.line_id, valid_stations)

				if route1 is not None and route2 is not None:
					line_id_to_routes[line.line_id] = (route1, route2)

	return line_id_to_routes
def test_get_connection_managers_info(
        api_backend,
        api_test_context,
        api_raiden_service):

    # check that no connection managers exists yet
    request = grequests.get(
        api_url_for(api_backend, 'connectionmanagersresource')
    )
    response = request.send().response
    token_addresses = response.json()
    assert token_addresses == dict()

    funds = 100
    token_address1 = '0xea674fdde714fd979de3edf0f56aa9716b898ec8'
    connect_data_obj = {
        'funds': funds,
    }
    request = grequests.put(
        api_url_for(api_backend, 'connectionsresource', token_address=token_address1),
        json=connect_data_obj,
    )
    response = request.send().response
    assert_no_content_response(response)

    # check that there now is one registered channel manager
    request = grequests.get(
        api_url_for(api_backend, 'connectionmanagersresource')
    )
    response = request.send().response
    token_addresses = response.json()
    assert isinstance(token_addresses, dict) and len(token_addresses.keys()) == 1
    assert token_address1 in token_addresses
    assert isinstance(token_addresses[token_address1], dict)
    assert set(token_addresses[token_address1].keys()) == {'funds', 'sum_deposits', 'channels'}

    funds = 100
    token_address2 = '0x3edf0f56aa9716b898ec8ea674fdde714fd979de'
    connect_data_obj = {
        'funds': funds,
    }
    request = grequests.put(
        api_url_for(api_backend, 'connectionsresource', token_address=token_address2),
        json=connect_data_obj,
    )
    response = request.send().response
    assert_no_content_response(response)

    # check that there now are two registered channel managers
    request = grequests.get(
        api_url_for(api_backend, 'connectionmanagersresource')
    )
    response = request.send().response
    token_addresses = response.json()
    assert isinstance(token_addresses, dict) and len(token_addresses.keys()) == 2
    assert token_address2 in token_addresses
    assert isinstance(token_addresses[token_address2], dict)
    assert set(token_addresses[token_address2].keys()) == {'funds', 'sum_deposits', 'channels'}
def test_break_blockchain_events(
        api_backend,
        api_test_context,
        api_raiden_service,
        rest_api_port_number):

    api_test_context.add_events([{
        '_event_type': 'ChannelNew',
        'settle_timeout': 10,
        'netting_channel': '0xea674fdde714fd979de3edf0f56aa9716b898ec8',
        'participant1': '0x4894a542053248e0c504e3def2048c08f73e1ca6',
        'participant2': '0x356857Cd22CBEFccDa4e96AF13b408623473237A',
        'block_number': 15,
    }, {
        '_event_type': 'ChannelSettled',
        'block_number': 35,
    }])

    # Let's make sure that token_address as a query argument does not override
    # the provided token_address
    api_test_context.specify_token_for_channelnew('0x61c808d82a3ac53231750dadc13c777b59310bd9')
    request = grequests.get(
        'http://localhost:{port}/api/1/events/tokens/0x61c808d82a3ac53231750dadc13c777b59310bd9'
        '?from_block=5&to_block=20'
        '&token_address=0x167a9333bf582556f35bd4d16a7e80e191aa6476'.format(
            port=rest_api_port_number,
        )
    )
    response = request.send().response
    assert_proper_response(response)
    response = json.loads(response.text)
    assert len(response) == 1
    assert response[0] == {
        'event_type': 'ChannelNew',
        'settle_timeout': 10,
        'netting_channel': '0xea674fdde714fd979de3edf0f56aa9716b898ec8',
        'participant1': '0x4894a542053248e0c504e3def2048c08f73e1ca6',
        'participant2': '0x356857Cd22CBEFccDa4e96AF13b408623473237A',
        'block_number': 15,
    }

    # Assert the same for the event/channels endpoint
    api_test_context.specify_channel_for_events('0xedbaf3c5100302dcdda53269322f3730b1f0416d')
    request = grequests.get(
        'http://localhost:{port}/api/1/events/channels/0xedbaf3c5100302dcdda53269322f3730b1f0416d'
        '?from_block=10&to_block=90'
        '&channel_address=0x167A9333BF582556f35Bd4d16A7E80E191aa6476'.format(
            port=rest_api_port_number,
        )
    )
    response = request.send().response
    assert_proper_response(response)
    response = json.loads(response.text)
    assert len(response) == 1
    assert response[0] == {
        'event_type': 'ChannelSettled',
        'block_number': 35,
    }
Example #17
0
 def test_map_timeout_no_exception_handler(self):
     """
     compliance with existing 0.2.0 behaviour
     """
     reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))]
     responses = grequests.map(reqs)
     self.assertIsNone(responses[0])
     self.assertTrue(responses[1].ok)
     self.assertEqual(len(responses), 2)
Example #18
0
def reap(file_name):
    config = ConfigParser.SafeConfigParser(allow_no_value=False)
    cfg_success = config.read('combine.cfg')
    if not cfg_success:
        logger.error('Reaper: Could not read combine.cfg.')
        logger.error('HINT: edit combine-example.cfg and save as combine.cfg.')
        return

    inbound_url_file = config.get('Reaper', 'inbound_urls')
    outbound_url_file = config.get('Reaper', 'outbound_urls')

    try:
        with open(inbound_url_file, 'rb') as f:
    	    inbound_urls = [url.rstrip('\n') for url in f.readlines()]
    except EnvironmentError as e:
        logger.error('Reaper: Error while opening "%s" - %s' % (inbound_url_file, e.strerror))
        return

    try:
        with open(outbound_url_file, 'rb') as f:
            outbound_urls = [url.rstrip('\n') for url in f.readlines()]
    except EnvironmentError as e:
        logger.error('Reaper: Error while opening "%s" - %s' % (outbound_url_file, e.strerror))
        return

    logger.info('Fetching inbound URLs')
    inbound_files=[]
    for url in inbound_urls:
        if url.startswith('file://'):
            inbound_files.append(url.partition('://')[2])
            inbound_urls.remove(url)
    headers = {'User-Agent': 'harvest.py'}
    reqs = [grequests.get(url, headers=headers) for url in inbound_urls]
    inbound_responses = grequests.map(reqs)
    inbound_harvest = [(response.url, response.status_code, response.text) for response in inbound_responses if response]
    for each in inbound_files:
        with open(each,'rb') as f:
            inbound_harvest.append(('file://'+each, 200, f.read()))

    logger.info('Fetching outbound URLs')
    outbound_files=[]
    for url in outbound_urls:
        if url.startswith('file://'):
            outbound_files.append(url.partition('://')[2])
            outbound_urls.remove(url)
    reqs = [grequests.get(url, headers=headers) for url in outbound_urls]
    outbound_responses = grequests.map(reqs)
    outbound_harvest = [(response.url, response.status_code, response.text) for response in outbound_responses if response]
    for each in outbound_files:
        with open(each,'rb') as f:
            outbound_harvest.append(('file://'+each, 200, f.read()))

    logger.error('Storing raw feeds in %s' % file_name)
    harvest = {'inbound': inbound_harvest, 'outbound': outbound_harvest}

    with open(file_name, 'wb') as f:
        json.dump(harvest, f, indent=2)
Example #19
0
 def test_map_timeout_exception_handler_returns_false(self):
     """
     if you don't want your exceptions to show up in the map result
     """
     def exception_handler(request, exception):
         return False
     reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))]
     responses = grequests.map(reqs, exception_handler=exception_handler)
     self.assertTrue(responses[0].ok)
     self.assertEqual(len(responses), 1)
Example #20
0
 def test_map_timeout_exception_handler_returns_exception(self):
     """
     ensure returned value from exception handler is stuffed in the map result
     """
     def exception_handler(request, exception):
         return exception
     reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))]
     responses = grequests.map(reqs, exception_handler=exception_handler)
     self.assertIsInstance(responses[0], Timeout)
     self.assertTrue(responses[1].ok)
     self.assertEqual(len(responses), 2)
Example #21
0
 def test_map_timeout_exception_handler_no_return(self):
     """
     ensure default behaviour for a handler that returns None
     """
     def exception_handler(request, exception):
         pass
     reqs = [grequests.get(httpbin('delay/1'), timeout=0.001), grequests.get(httpbin('/'))]
     responses = grequests.map(reqs, exception_handler=exception_handler)
     self.assertIsNone(responses[0])
     self.assertTrue(responses[1].ok)
     self.assertEqual(len(responses), 2)
    def main(self, server, port):
        logging.basicConfig(filename='logs/reader.log', level=logging.DEBUG)
        logging.info("Attempting to set up the reader on: %s:%d" % (server,
                                                                    port))
        try:
            """
                ls /dev/serial/by-id/ to get all connected devices.
                Use that to determine this
                /dev/serial/by-id/usb-FTDI_FT232R_USB_UART_AD026CI5-if00-port0
            """
            usb_devices = glob.glob("/dev/serial/by-id/*")
            ser = serial.Serial(usb_devices[0], 9600, timeout=0)
            logging.info("Crosstag reader is up and running.")
            sys.stdout.flush()
        except:
            now = datetime.now()
            logging.error('%s reader ERROR unable to setup the RFID reade,",\
                          " is it plugged in?' % (now))
            sys.exit(-1)
        while True:
            if ser.inWaiting() > 0:
                data = ser.readline()
                if len(data) < 12:
                    continue
                now = datetime.now()
                ser.flushOutput()
                data = data.strip()
                tag_nbr = data[1:]
                if len(tag_nbr) != 12:
                    logging.error('%s reader ERROR [%s] is too long.",\
                                  " len(tag_nbr): %d' % (now, tag_nbr,
                                                         len(tag_nbr)))
                    # TODO, PiFace code to ask the person to tag again.
                    # audio signal and visual signal.
                    continue
                try:
                    print('%s reader tagging [%s]' % (now, tag_nbr))
                    urls = ["http://%s:%d/crosstag/v1.0/tagevent/%s" % (server, port, tag_nbr)]

                    unsent = (grequests.get(url) for url in urls)
                    res = grequests.map(unsent)
                    res = grequests.get("http://%s:%d/crosstag/v1.0/tagevent/%s" % (server, port, tag_nbr), timeout=3)

                    now = datetime.now()



                    logging.info("%s reader tagging result: [%s]" % (now, tag_nbr))
                except:  # catch *all* exceptions
                    e = sys.exc_info()[0]
                    logging.error('%s reader failed (timeout?) to tag %s error: %s' % (now, tag_nbr, e))
            else:
                time.sleep(1)
Example #23
0
def settings(request):

    if request.method == 'POST':

        # Update global settings for notifications.
        postdata = {
            'sendEmail': bool(request.POST.get('profile_receive_email')),
            'sendSMS': False,
        }
        try:
            GlobalOptions.update(request.get_host(), postdata, request.access_token)
        except APIException as exc:
            return respond_with_error(request, exc)

        # Update settings for each notification type.
        for x in range(1, 10):
            if request.POST.get('id_' + str(x)):
                postdata = {
                    'id': int(request.POST['id_' + str(x)]),
                    'sendEmail': bool(request.POST.get('send_email_' + str(x))),
                    'sendSMS': False,
                    }
                try:
                    UpdatePreference.update(request.get_host(), request.POST['id_' + str(x)],
                        postdata, request.access_token)
                except APIException as exc:
                    return respond_with_error(request, exc)

        return HttpResponseRedirect(reverse('updates-settings'))

    if request.method == 'GET':
        url, params, headers = UpdatePreference.build_request(request.get_host(), request.access_token)
        request.view_requests.append(grequests.get(url, params=params, headers=headers))

        url2, params2, headers2 = GlobalOptions.build_request(request.get_host(), request.access_token)
        request.view_requests.append(grequests.get(url2, params=params2, headers=headers2))

        try:
            responses = response_list_to_dict(grequests.map(request.view_requests))
        except APIException as exc:
            return respond_with_error(request, exc)

        preference_list = UpdatePreference.from_list(responses[url])
        global_options = GlobalOptions.from_api_response(responses[url2])

        view_data = {
            'user': Profile(responses[request.whoami_url], summary=False),
            'site': Site(responses[request.site_url]),
            'content': preference_list,
            'globaloptions': global_options,
        }
        return render(request, settings_template, view_data)
Example #24
0
def confirm(request):
    """
    View for moderation actions on a single item.
    """

    if request.method == 'GET':
        if request.GET.get('item_type') == 'conversation':
            url, params, headers = Conversation.build_request(
                request.get_host(),
                request.GET.get('item_id'),
                access_token=request.access_token
            )
            request.view_requests.append(grequests.get(url, params=params, headers=headers))
            responses = response_list_to_dict(grequests.map(request.view_requests))
            content = Conversation.from_api_response(responses[url])

        elif request.GET.get('item_type') == 'event':
            url, params, headers = Event.build_request(
                request.get_host(),
                request.GET.get('item_id'),
                access_token=request.access_token
            )
            request.view_requests.append(grequests.get(url, params=params, headers=headers))
            responses = response_list_to_dict(grequests.map(request.view_requests))
            content = Event.from_api_response(responses[url])

        elif request.GET.get('item_type') == 'microcosm':
            url, params, headers = Microcosm.build_request(
                request.get_host(),
                request.GET.get('item_id'),
                access_token=request.access_token
            )
            request.view_requests.append(grequests.get(url, params=params, headers=headers))
            responses = response_list_to_dict(grequests.map(request.view_requests))
            content = Microcosm.from_api_response(responses[url])

        view_data = {
            'user': Profile(responses[request.whoami_url], summary=False),
            'site': Site(responses[request.site_url]),
            'content': content,
            'item_type': request.GET.get('item_type'),
            'action': request.GET.get('action'),
        }

        if request.GET.get('action') == 'move':
            # Fetch list of microcosms to supply in form.
            view_data['microcosms'] = MicrocosmList.retrieve(
                request.get_host(),
                access_token=request.access_token
            )

        return render(request, 'forms/moderation_item.html', view_data)
Example #25
0
def edit_members(request, microcosm_id, group_id):

    if request.method == 'POST':
        pass
    elif request.method == 'GET':
        try:
            offset = int(request.GET.get('offset', 0))
        except ValueError:
            offset = 0

        microcosm_url, params, headers = Microcosm.build_request(request.get_host(), id=microcosm_id,
            offset=offset, access_token=request.access_token)
        request.view_requests.append(grequests.get(microcosm_url, params=params, headers=headers))

        role_url, params, headers = Role.build_request(request.get_host(), microcosm_id=microcosm_id,
            id=group_id, offset=offset, access_token=request.access_token)
        request.view_requests.append(grequests.get(role_url, params=params, headers=headers))

        criteria_url, params, headers = RoleCriteriaList.build_request(request.get_host(),
            microcosm_id=microcosm_id, id=group_id, offset=offset, access_token=request.access_token)
        request.view_requests.append(grequests.get(criteria_url, params=params, headers=headers))

        profiles_url, params, headers = RoleProfileList.build_request(request.get_host(),
            microcosm_id=microcosm_id, id=group_id, offset=offset, access_token=request.access_token)
        request.view_requests.append(grequests.get(profiles_url, params=params, headers=headers))

        try:
            responses = response_list_to_dict(grequests.map(request.view_requests))
        except APIException as exc:
            return respond_with_error(request, exc)

        microcosm = Microcosm.from_api_response(responses[microcosm_url])
        role = Role.from_api_response(responses[role_url])
        criteria = RoleCriteriaList(responses[criteria_url])
        profiles = RoleProfileList(responses[profiles_url])

        view_data = {
            'user': Profile(responses[request.whoami_url], summary=False) if request.whoami_url else None,
            'site': Site(responses[request.site_url]),
            'site_section': 'memberships',
            'content': microcosm,
            'role': role,
            'criteria': criteria,
            'profiles': profiles,
            'item_type': 'memberships',
            'state_edit': True,
            'pagination': build_pagination_links(responses[microcosm_url]['items']['links'], microcosm.items)
        }

        return render(request, members_form_template, view_data)
Example #26
0
def test_api_get_channel_list(
        api_backend,
        token_addresses,
        reveal_timeout,
):
        partner_address = '0x61C808D82A3Ac53231750daDc13c777b59310bD9'

        request = grequests.get(
            api_url_for(
                api_backend,
                'channelsresource',
            ),
        )
        response = request.send().response
        assert_proper_response(response, HTTPStatus.OK)
        assert response.json() == []

        # let's create a new channel
        token_address = token_addresses[0]
        settle_timeout = 1650
        channel_data_obj = {
            'partner_address': partner_address,
            'token_address': to_checksum_address(token_address),
            'settle_timeout': settle_timeout,
            'reveal_timeout': reveal_timeout,
        }

        request = grequests.put(
            api_url_for(
                api_backend,
                'channelsresource',
            ),
            json=channel_data_obj,
        )
        response = request.send().response

        assert_proper_response(response, HTTPStatus.CREATED)

        request = grequests.get(
            api_url_for(
                api_backend,
                'channelsresource',
            ),
        )
        response = request.send().response
        assert_proper_response(response, HTTPStatus.OK)
        channel_info = response.json()[0]
        assert channel_info['partner_address'] == partner_address
        assert channel_info['token_address'] == to_checksum_address(token_address)
        assert 'token_network_identifier' in channel_info
Example #27
0
def bulkRequests(topic_list):
	start_time = time.time()
	debugFunction()

	try:
		rs = (grequests.get(u, timeout=5) for u in topic_list)
		response = grequests.map(rs)
		return response
	except:
		s = (grequests.get(u, timeout=5) for u in topic_list)
		response = grequests.map(rs)
		return response
	
	elapsedTime(start_time)
	return
def get_information_from_page(url_list,asynchronous=False):
    
    if asynchronous:
        for urls in url_list:
            rs = (grequests.get(u,stream=False) for u in urls)
            responses = grequests.map(rs)
            results = []
            for r in responses:
                result = {}
                html = lxml.html.fromstring(r.text)
                posting_body = html.xpath('//div[@class="postingBody"]')
                result["textbody"] = [i.text_content() for i in posting_body]
                result['pictures'] = html.xpath('//ul[@id="viewAdPhotoLayout"]/li/a/@href')
                result['url'] = r.url
                results.append(result)
                r.close()
        return results
            
    else:
        r = requests.get(url_list)
        html = lxml.html.fromstring(r.text)
        posting_body = html.xpath('//div[@class="postingBody"]')
        textbody = [i.text_content() for i in posting_body]
        pictures = html.xpath('//ul[@id="viewAdPhotoLayout"]/li/a/@href')
        return textbody,pictures
Example #29
0
def list(request):
    # Offset for paging of huddles
    try:
        offset = int(request.GET.get('offset', 0))
    except ValueError:
        offset = 0

    huddle_url, params, headers = HuddleList.build_request(request.get_host(), offset=offset,
        access_token=request.access_token)

    request.view_requests.append(grequests.get(huddle_url, params=params, headers=headers))
    try:
        responses = response_list_to_dict(grequests.map(request.view_requests))
    except APIException as exc:
        return respond_with_error(request, exc)

    huddles = HuddleList(responses[huddle_url])

    view_data = {
        'user': Profile(responses[request.whoami_url], summary=False) if request.whoami_url else None,
        'site': Site(responses[request.site_url]),
        'content': huddles,
        'pagination': build_pagination_links(responses[huddle_url]['huddles']['links'], huddles.huddles)
    }
    return render(request, list_template, view_data)
Example #30
0
    def process_request(self, request):
        """
        Checks for access_token cookie and appends it to the request object if present.

        All request objects have a view_requests attribute which is a list of requests
        that will be executed by grequests to fetch data for the view.
        """

        request.access_token = None
        request.whoami_url = ''
        request.view_requests = []
        request.site = None

        if request.COOKIES.has_key('access_token'):
            request.access_token = request.COOKIES['access_token']
            url, params, headers = WhoAmI.build_request(request.get_host(), request.access_token)
            request.view_requests.append(grequests.get(url, params=params, headers=headers))
            request.whoami_url = url

        try:
            request.site = Site.retrieve(request.get_host())
        except APIException as e:
            if e.status_code == 400:
                return HttpResponseRedirect('https://microco.sm')
        return None
Example #31
0
    def 模具一一gr无序网址列表请求返回网页内容(self, 网址列表, 分流数=100,并发数=20):  # grequests.imap(任务列

        初始网址列表数 = len(网址列表)

        条件循环 = 1

        返回网页一链接组列表 = []

        while 条件循环 == 1:

            此时数 = int(time.time())
            if 此时数 > self.换IP时间计数 + 60:
                self.模具一一高位换头部信息()
                self.模具一一换ip连接()
            网址列表数 = len(网址列表)
            任务列表 = []
            网址分列表 = []
            for 各帖子链接 in 网址列表[0:分流数]:
                网址分列表.append(各帖子链接)

            if len(网址列表) > 分流数 - 1:  # break # 结束循环 continue # 跳过当前循环,继续进行下一轮循环
                网址列表 = 网址列表[分流数:]
            else:  # 否则
                网址列表 = []

            for 各帖子链接 in 网址分列表:
                任务 = grequests.get(各帖子链接, headers=self.头部信息)  # timeout=len(任务列表)//2,
                任务列表.append(任务)

            try:  # 调用异常处理,应对易发生错误的位置
                返回网页内容集 = grequests.imap(任务列表, size=并发数)  # size=3 并发数 3  gtimeout超时时间
            except (requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout,
                    requests.exceptions.ConnectionError, requests.exceptions.ConnectTimeout,
                    requests.exceptions.ChunkedEncodingError, requests.exceptions.InvalidSchema) as 异常:

                print('gr无序异常,倒数6秒再连接', 异常)
                # 网址列表.append(返回网页内容.url)

                time.sleep(3)
            else:

                for 返回网页内容 in 返回网页内容集:
                    网页内容文本 = str(返回网页内容)

                    if '<Response [200]>' in 网页内容文本 and "访问过于频繁" not in 返回网页内容.text:
                        # print('返回网页内容', 返回网页内容)

                        返回网页内容.encoding = "UTF-8"

                        返回网页一链接组 = []
                        返回网页一链接组.append(返回网页内容)
                        返回网页一链接组.append(返回网页内容.url)
                        返回网页一链接组列表.append(返回网页一链接组)

                        print('成功采集网页:', 返回网页内容.url)

                    else:  # 否则
                        网址列表.append(返回网页内容.url)

                此时数 = int(time.time())
                if len(网址列表) == 0:
                    print('网页采集,全部完成')

                    条件循环 = 998
                elif 此时数 > self.换IP时间计数 + 初始网址列表数 * 3:
                    print('网页未完成采集数:', 初始网址列表数 - len(返回网页一链接组列表))

                    条件循环 = 998

                #time.sleep(2)  # 等待

        return 返回网页一链接组列表  # 返回
Example #32
0
This works faster than the solution in the book by making asynchronous 
requests with grequests (https://github.com/spyoungtech/grequests)
'''
import grequests
from urllib.request import urlopen
from bs4 import BeautifulSoup

#get the url
page_url = input("Which web page do you want to analyze? ")
try:
    page = urlopen(page_url)
except:
    print("Cannot open %s" % page_url)
    quit()

#get all the hlinks
soup = BeautifulSoup(page, features="lxml")

links = [(link["href"]) for link in soup.find_all("a") if link.has_attr("href")]

# This will notify about and print broken links
def exception_handler(request, exception):
    print("This link might be broken:")
    print(exception)

# Create a set of unsent Requests:
rs = (grequests.get(u) for u in links)

# Send them all at the same time:
grequests.map(rs, exception_handler=exception_handler)
Example #33
0
def query_get_task_with_details(bot_memo, present_skill, bot_nlp):

    if ((bot_memo == {} or bot_memo['index']) and present_skill == 'get_task'):

        #requests can be used for synchronous requests
        # r = requests.get("https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json", auth=HTTPBasicAuth('pritamsa', 'rupu@0801'))
        # body1 = r.json()

        #grequests is faster
        url1 = [
            "https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json"
        ]
        rs1 = (grequests.get(u, auth=('pritamsa', 'rupu@0801')) for u in url1)
        #both imap and map can be used
        #reque = grequests.imap(rs,size=1)
        reque1 = grequests.map(rs1, size=1)
        response_array1 = []
        for response1 in reque1:
            print(response1)
            x1 = response1.json()
            response_array1.append(x1)
        body1 = response_array1[0]

        no_of_tasks = len(body1["d"]["results"])
        if (body1["d"]["results"]):
            #task details
            instance_id = body1["d"]["results"][0]["InstanceID"]
            task_title = body1["d"]["results"][0]["TaskTitle"]

            scrapped_po_no = task_title.split("order ", 1)[1]

            body2, body3 = take_action_async(scrapped_po_no)

            #po_header detail
            created_by_user = body2["d"]["CreatedByUser"]
            SupplierName = body2["d"]["SupplierName"]
            PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]
            DocumentCurrency = body2["d"]["DocumentCurrency"]
            PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]

            final_reply_string = ''
            concat_string_for_multiple_lineitems = ''
            per_item_desc_dict = {}
            all_item_details = {}
            #po item detail
            no_of_line_items = len(body3["d"]["results"])
            for i in range(no_of_line_items):
                Material = body3["d"]["results"][i]["Material_Text"]
                Plant = body3["d"]["results"][i]["Plant"]
                OrderQuantity = body3["d"]["results"][i]["OrderQuantity"]
                netPriceItem = body3["d"]["results"][i]["NetPriceAmount"]
                documentCurrency = body3["d"]["results"][i]["DocumentCurrency"]
                price_present_item_with_currency = netPriceItem + documentCurrency

                item_no = 'item : ' + str(i + 1)
                # print(item_no)
                #item_no = dict(item_no)
                per_item_desc_dict = {
                    item_no: {
                        'Material': Material,
                        'Plant': Plant,
                        'OrderQuantity': OrderQuantity,
                        'netPriceItem': price_present_item_with_currency
                    }
                }
                all_item_details.update(per_item_desc_dict)

                #use this when sending the item details as string all in one reply
                # concat_string_for_multiple_lineitems = concat_string_for_multiple_lineitems \
                #     + 'Material: ' + Material + '.\n' + 'plant: ' + Plant + '.\n' \
                #     + 'OrderQuantity: ' + OrderQuantity + '.\n'

            get_task_string = ''
            get_task_string_with_header_detail = ''

            get_task_string = task_title + '.' + '\n'

            get_task_string_with_header_detail = 'created by user: '******'.' + '\n' + 'SupplierName: ' + SupplierName \
                    + '.' + '\n' + 'PurchaseOrderNetAmount: ' + PurchaseOrderNetAmount + ' ' + DocumentCurrency + '.'+'\n'

            #final_reply_string = 'Now you have got, '+ str(no_of_tasks) + ' pending tasks to approve. ' + get_task_string + get_task_string_with_header_detail +'You have: ' + str(no_of_line_items) +' items.\n'+ concat_string_for_multiple_lineitems + " say approve to approve this task or say ignore to skip this task and move on to your next task, or say next to get your next task with details."
            final_reply_string = 'Now you have got, ' + str(
                no_of_tasks
            ) + ' pending tasks to approve. ' + get_task_string + get_task_string_with_header_detail + 'You have: ' + str(
                no_of_line_items
            ) + ' items.\n' + " say get item details to get all the item details in this purchase order. Or,say approve to approve this task or say ignore to skip this task and move on to your next task, or say next to get your next task with details."

            return final_reply_string, 1, instance_id, created_by_user, SupplierName, (
                PurchaseOrderNetAmount + ' ' + DocumentCurrency
            ), '', all_item_details, no_of_line_items, scrapped_po_no  #return 1for memory index as no memo is present in the beggining

        else:
            final_reply_string = 'no more tasks to approve in your inbox.'
            return final_reply_string, 1, bot_memo, bot_memo, bot_memo, bot_memo, '', '', '', bot_memo

    elif (
        (bot_memo['index']) and
        (present_skill == 'get_next_task' or present_skill == 'ignore_task')):
        #requests can be used for synchronous requests
        # r = requests.get("https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json", auth=HTTPBasicAuth('pritamsa', 'rupu@0801'))
        # body1 = r.json()

        #grequests is faster
        url1 = [
            "https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json"
        ]
        rs1 = (grequests.get(u, auth=('pritamsa', 'rupu@0801')) for u in url1)
        #both imap and map can be used
        #reque = grequests.imap(rs,size=1)
        reque1 = grequests.map(rs1, size=1)
        response_array1 = []
        for response1 in reque1:
            print(response1)
            x1 = response1.json()
            response_array1.append(x1)
        body1 = response_array1[0]
        no_of_tasks = len(body1["d"]["results"])
        if ((len(body1["d"]["results"]) == 1)):

            instance_id = body1["d"]["results"][0]["InstanceID"]
            task_title = body1["d"]["results"][0]["TaskTitle"]

            scrapped_po_no = task_title.split("order ", 1)[1]

            body2, body3 = take_action_async(scrapped_po_no)

            #po_header detail
            created_by_user = body2["d"]["CreatedByUser"]
            SupplierName = body2["d"]["SupplierName"]
            PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]
            DocumentCurrency = body2["d"]["DocumentCurrency"]
            PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]

            final_reply_string = ''
            concat_string_for_multiple_lineitems = ''
            per_item_desc_dict = {}
            all_item_details = {}

            #po item detail
            no_of_line_items = len(body3["d"]["results"])
            for i in range(no_of_line_items):
                Material = body3["d"]["results"][i]["Material_Text"]
                Plant = body3["d"]["results"][i]["Plant"]
                OrderQuantity = body3["d"]["results"][i]["OrderQuantity"]
                netPriceItem = body3["d"]["results"][i]["NetPriceAmount"]
                documentCurrency = body3["d"]["results"][i]["DocumentCurrency"]
                price_present_item_with_currency = netPriceItem + documentCurrency

                item_no = 'item : ' + str(i + 1)
                # print(item_no)
                #item_no = dict(item_no)
                per_item_desc_dict = {
                    item_no: {
                        'Material': Material,
                        'Plant': Plant,
                        'OrderQuantity': OrderQuantity,
                        'netPriceItem': price_present_item_with_currency
                    }
                }
                all_item_details.update(per_item_desc_dict)

                #use this when sending the item details as string all in one reply
                # concat_string_for_multiple_lineitems = concat_string_for_multiple_lineitems \
                #     + 'Material: ' + Material + '.\n' + 'plant: ' + Plant + '.\n' \
                #     + 'OrderQuantity: ' + OrderQuantity + '.\n'

            get_task_string = ''
            get_task_string_with_header_detail = ''

            get_task_string = task_title + '.' + '\n'

            get_task_string_with_header_detail = 'created by user: '******'.' + '\n' + 'SupplierName: ' + SupplierName \
                    + '.' + '\n' + 'PurchaseOrderNetAmount: ' + PurchaseOrderNetAmount + ' ' + DocumentCurrency + '.'+'\n'

            # final_reply_string = 'Now you have got, '+ str(no_of_tasks) + ' pending tasks to approve. ' + get_task_string + get_task_string_with_header_detail +'You have: ' + str(no_of_line_items) +' items.\n'+ concat_string_for_multiple_lineitems + " say approve to approve this task or say ignore to skip this task and move on to your next task, or say next to get your next task with details."
            final_reply_string = 'Now you have got, ' + str(
                no_of_tasks
            ) + ' pending tasks to approve. ' + get_task_string + get_task_string_with_header_detail + 'You have: ' + str(
                no_of_line_items
            ) + ' items.\n' + " say get item details to get all the item details in this purchase order. Or,say approve to approve this task or say ignore to skip this task and move on to your next task, or say next to get your next task with details."

            return final_reply_string, 1, instance_id, created_by_user, SupplierName, (
                PurchaseOrderNetAmount + ' ' + DocumentCurrency
            ), '', all_item_details, no_of_line_items, scrapped_po_no  #return 1for memory index as no memo is present in the beggining

        elif ((len(body1["d"]["results"]) > 1)
              and bot_memo['index'] < len(body1["d"]["results"])):
            #task details
            instance_id = body1["d"]["results"][
                bot_memo['index']]["InstanceID"]
            task_title = task_title = body1["d"]["results"][
                bot_memo['index']]["TaskTitle"]
            #print(task_title)
            scrapped_po_no = task_title.split("order ", 1)[1]
            #print(scrapped_po_no)

            body2, body3 = take_action_async(scrapped_po_no)

            #po_header detail
            created_by_user = body2["d"]["CreatedByUser"]
            SupplierName = body2["d"]["SupplierName"]
            PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]
            DocumentCurrency = body2["d"]["DocumentCurrency"]
            PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]

            final_reply_string = ''
            concat_string_for_multiple_lineitems = ''
            per_item_desc_dict = {}
            all_item_details = {}

            #po item detail
            #only show one or two tasks
            no_of_line_items = len(body3["d"]["results"])
            for i in range(no_of_line_items):
                Material = body3["d"]["results"][i]["Material_Text"]
                Plant = body3["d"]["results"][i]["Plant"]
                OrderQuantity = body3["d"]["results"][i]["OrderQuantity"]
                netPriceItem = body3["d"]["results"][i]["NetPriceAmount"]
                documentCurrency = body3["d"]["results"][i]["DocumentCurrency"]
                price_present_item_with_currency = netPriceItem + documentCurrency

                item_no = 'item : ' + str(i + 1)
                # print(item_no)
                #item_no = dict(item_no)
                per_item_desc_dict = {
                    item_no: {
                        'Material': Material,
                        'Plant': Plant,
                        'OrderQuantity': OrderQuantity,
                        'netPriceItem': price_present_item_with_currency
                    }
                }
                all_item_details.update(per_item_desc_dict)

                #use this when sending the item details as string all in one reply
                # concat_string_for_multiple_lineitems = concat_string_for_multiple_lineitems \
                #     + 'Material: ' + Material + '.\n' + 'plant: ' + Plant + '.\n' \
                #     + 'OrderQuantity: ' + OrderQuantity + '.\n'

            get_task_string = ''
            get_task_string_with_header_detail = ''

            get_task_string = task_title + '.' + '\n'

            get_task_string_with_header_detail = 'created by user: '******'.' + '\n' + 'SupplierName: ' + SupplierName \
                    + '.' + '\n' + 'PurchaseOrderNetAmount: ' + PurchaseOrderNetAmount + ' ' + DocumentCurrency + '.'+'\n'

            # final_reply_string = get_task_string + get_task_string_with_header_detail +'You have: ' + str(no_of_line_items) +' items in this P.O.\n'+ concat_string_for_multiple_lineitems + " say approve to approve this task or say ignore to skip this task and move on to your next task, or say next to get your next task with details."
            final_reply_string = 'Now you have got, ' + str(
                no_of_tasks
            ) + ' pending tasks to approve. ' + get_task_string + get_task_string_with_header_detail + 'You have: ' + str(
                no_of_line_items
            ) + ' items.\n' + " say get item details to get all the item details in this purchase order. Or,say approve to approve this task or say ignore to skip this task and move on to your next task, or say next to get your next task with details."

            #print(get_task_string)

            #print(final_reply_string)
            return final_reply_string, bot_memo[
                'index'] + 1, instance_id, created_by_user, SupplierName, (
                    PurchaseOrderNetAmount + ' ' + DocumentCurrency
                ), '', all_item_details, no_of_line_items, scrapped_po_no

        elif (len(body1["d"]["results"]) > 0) and (bot_memo['index'] >= len(
                body1["d"]["results"])):

            final_reply_string = 'no more tasks to approve in your inbox.'
            return final_reply_string, bot_memo['index'], len(
                body1["d"]["results"]
            ), bot_memo['created_by'], bot_memo['SupplierName'], bot_memo[
                'PurchaseOrderNetAmount'], '', '', '', bot_memo[
                    'scrapped_po_no']

        else:

            final_reply_string = 'I think there are no more pending approvals for you. Say, "get my tasks", to get your pending approvals.'
            return final_reply_string, bot_memo['index'], len(
                body1["d"]["results"]
            ), bot_memo['created_by'], bot_memo['SupplierName'], bot_memo[
                'PurchaseOrderNetAmount'], '', '', '', bot_memo[
                    'scrapped_po_no']

    #repeat intent is handled via bot memory not via code

    # elif((bot_memo['index']) and present_skill == 'repeat'):

    #     r = requests.get("https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json", auth=HTTPBasicAuth('pritamsa', 'rupu@0801'))
    #     body1 = r.json()
    #     if (body1["d"]["results"] and bot_memo['index'] <= len(body1["d"]["results"])):
    #         #task details
    #         instance_id = body1["d"]["results"][bot_memo['index']-1]["InstanceID"]
    #         task_title = body1["d"]["results"][bot_memo['index']-1]["TaskTitle"]

    #         scrapped_po_no = task_title.split("order ",1)[1]

    #         body2,body3 = take_action_async(scrapped_po_no)

    #         #po_header detail
    #         created_by_user = body2["d"]["CreatedByUser"]
    #         SupplierName = body2["d"]["SupplierName"]
    #         PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]
    #         DocumentCurrency = body2["d"]["DocumentCurrency"]
    #         PurchaseOrderNetAmount = body2["d"]["PurchaseOrderNetAmount"]

    #         final_reply_string = ''
    #         concat_string_for_multiple_lineitems = ''

    #         #po item detail
    #         #only show one or two tasks
    #         no_of_line_items = len(body3["d"]["results"])
    #         for i in range(no_of_line_items):
    #             Material = body3["d"]["results"][i]["Material_Text"]
    #             Plant = body3["d"]["results"][i]["Plant"]
    #             OrderQuantity = body3["d"]["results"][i]["OrderQuantity"]

    #             concat_string_for_multiple_lineitems = concat_string_for_multiple_lineitems \
    #                 + 'Material: ' + Material + '.\n' + 'plant: ' + Plant + '.\n' \
    #                 + 'OrderQuantity: ' + OrderQuantity + '.\n'

    #         get_task_string = ''
    #         get_task_string_with_header_detail = ''

    #         get_task_string = task_title + '\n'

    #         get_task_string_with_header_detail = 'created_by_user: '******'.' +'\n' + 'SupplierName: ' + SupplierName \
    #              +'.'   + '\n' + 'PurchaseOrderNetAmount: ' + PurchaseOrderNetAmount + ' ' + DocumentCurrency + '.' +'\n'

    #         final_reply_string = get_task_string + get_task_string_with_header_detail +'You have: ' + str(no_of_line_items) +' items\n'+ concat_string_for_multiple_lineitems + " say approve to approve this task or say ignore to skip this task and move on to your next task, or say next to get your next task with details."
    #         #print(get_task_string)

    #         #print(final_reply_string)
    #         return final_reply_string,bot_memo['index'],instance_id,created_by_user,SupplierName, (PurchaseOrderNetAmount + ' ' + DocumentCurrency)

    #     elif(body1["d"]["results"] and bot_memo['index'] >= len(body1["d"]["results"])):
    #         final_reply_string = 'no more tasks to approve...'
    #         return final_reply_string,bot_memo['index'],len(body1["d"]["results"]),created_by_user,SupplierName, (PurchaseOrderNetAmount + ' ' + DocumentCurrency)

    #     else:
    #         final_reply_string = 'I am facing some issues now please try later'
    #         return final_reply_string,bot_memo['index'],len(body1["d"]["results"]),created_by_user,SupplierName, (PurchaseOrderNetAmount + ' ' + DocumentCurrency)

    elif ((bot_memo['index']) and present_skill == 'approve'):
        after_approval_reply = 'successfully approved, please say,"get my tasks", to get your previous pending aapprovals from the beggining, or, say next to move on to your next task.'
        approval_failure_reply = "there was an issue with the server, Please try again later to approve..."
        session = requests.Session()
        header = {'x-csrf-token': 'Fetch'}
        present_task_instance_id = bot_memo['instanceID']

        # response = session.head("https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json", auth=HTTPBasicAuth('pritamsa', 'rupu@0801'),headers=header)
        # if (response.status_code != 200):
        #     return approval_failure_reply ,bot_memo['index'],present_task_instance_id,bot_memo['created_by'],bot_memo['SupplierName'], bot_memo['PurchaseOrderNetAmount'],approval_failure_reply,'','',bot_memo['scrapped_po_no']
        # elif (response.status_code == 200):
        #     cookie = session.cookies.get_dict()
        #     print(cookie)

        #     csrf = response.headers['x-csrf-token']
        #     #print(csrf)

        #     #post
        #     #approve
        #     header_2 = {'x-csrf-token':csrf}
        #     approve_po = session.post("https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/Decision?sap-client=400&SAP__Origin='S4HMYINBOCLNT200'&InstanceID="+ "'"+present_task_instance_id +"'""&DecisionKey='0001'&Comments='test%20approve'",auth=HTTPBasicAuth('pritamsa', 'rupu@0801'),headers=header_2,cookies=cookie)

        #     print('***************************************************************')
        #     print(approve_po.status_code)

        # approval request posted asynchronously
        url3 = [
            "https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json"
        ]
        head_res1 = (grequests.head(u,
                                    auth=('pritamsa', 'rupu@0801'),
                                    headers=header) for u in url3)
        #both imap and map can be used
        #reque = grequests.imap(rs,size=1)
        reque3 = grequests.map(head_res1, size=1)
        response_array3 = []
        for response3 in reque3:

            if (response3.status_code != 200):
                print("hey problem")
                return approval_failure_reply, bot_memo[
                    'index'], present_task_instance_id, bot_memo[
                        'created_by'], bot_memo['SupplierName'], bot_memo[
                            'PurchaseOrderNetAmount'], approval_failure_reply, '', '', bot_memo[
                                'scrapped_po_no']
            else:
                cookie = response3.cookies.get_dict()
                print(cookie)

                csrf = response3.headers['x-csrf-token']
                print(csrf)

                header_2 = {'x-csrf-token': csrf}

                url_post = [
                    "https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/Decision?sap-client=400&SAP__Origin='S4HMYINBOCLNT200'&InstanceID="
                    + "'" + present_task_instance_id + "'"
                    "&DecisionKey='0001'&Comments='test%20approve'"
                ]
                post_res = (grequests.post(u_post,
                                           auth=('pritamsa', 'rupu@0801'),
                                           headers=header_2,
                                           cookies=cookie)
                            for u_post in url_post)

                post_reque = grequests.map(post_res, size=1)
                response_array_post = []
                for response_post in post_reque:

                    if (response_post.status_code != 200):
                        print(
                            "hey problem in approving the request. Please try again later."
                        )
                        return approval_failure_reply, bot_memo[
                            'index'], present_task_instance_id, bot_memo[
                                'created_by'], bot_memo['SupplierName'], bot_memo[
                                    'PurchaseOrderNetAmount'], approval_failure_reply, '', '', bot_memo[
                                        'scrapped_po_no']

                    else:

                        return after_approval_reply, bot_memo[
                            'index'], present_task_instance_id, bot_memo[
                                'created_by'], bot_memo['SupplierName'], bot_memo[
                                    'PurchaseOrderNetAmount'], after_approval_reply, '', '', bot_memo[
                                        'scrapped_po_no']  #after this call the "next" task showing skill in bot

    elif ((bot_memo['index']) and present_skill == 'reject'):
        after_rejection_reply = 'successfully rejected, please say,"get my tasks", to get your previous pending aapprovals from the beggining, or, say next to move on to your next task.'
        rejection_failure_reply = "there was an issue with the server, Please try again later to approve..."
        session = requests.Session()
        header = {'x-csrf-token': 'Fetch'}
        present_task_instance_id = bot_memo['instanceID']

        # response = session.head("https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json", auth=HTTPBasicAuth('pritamsa', 'rupu@0801'),headers=header)
        # if (response.status_code != 200):
        #     return approval_failure_reply ,bot_memo['index'],present_task_instance_id,bot_memo['created_by'],bot_memo['SupplierName'], bot_memo['PurchaseOrderNetAmount'],approval_failure_reply,'','',bot_memo['scrapped_po_no']
        # elif (response.status_code == 200):
        #     cookie = session.cookies.get_dict()
        #     print(cookie)

        #     csrf = response.headers['x-csrf-token']
        #     #print(csrf)

        #     #post
        #     #approve
        #     header_2 = {'x-csrf-token':csrf}
        #     approve_po = session.post("https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/Decision?sap-client=400&SAP__Origin='S4HMYINBOCLNT200'&InstanceID="+ "'"+present_task_instance_id +"'""&DecisionKey='0001'&Comments='test%20approve'",auth=HTTPBasicAuth('pritamsa', 'rupu@0801'),headers=header_2,cookies=cookie)

        #     print('***************************************************************')
        #     print(approve_po.status_code)

        # approval request posted asynchronously
        url4 = [
            "https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/TaskCollection?sap-client=400&$filter=Status%20eq%20%27READY%27&$format=json"
        ]
        head_res4 = (grequests.head(u,
                                    auth=('pritamsa', 'rupu@0801'),
                                    headers=header) for u in url4)
        #both imap and map can be used
        #reque = grequests.imap(rs,size=1)
        reque4 = grequests.map(head_res4, size=1)
        response_array4 = []
        for response4 in reque4:

            if (response4.status_code != 200):
                print("hey problem")
                return rejection_failure_reply, bot_memo[
                    'index'], present_task_instance_id, bot_memo[
                        'created_by'], bot_memo['SupplierName'], bot_memo[
                            'PurchaseOrderNetAmount'], rejection_failure_reply, '', '', bot_memo[
                                'scrapped_po_no']
            else:
                cookie = response4.cookies.get_dict()
                print(cookie)

                csrf = response4.headers['x-csrf-token']
                print(csrf)

                header_2 = {'x-csrf-token': csrf}

                url_post = [
                    "https://p2001172697trial-trial.apim1.hanatrial.ondemand.com/p2001172697trial/Workflow_approval/Decision?sap-client=400&SAP__Origin='S4HMYINBOCLNT200'&InstanceID="
                    + "'" + present_task_instance_id + "'"
                    "&DecisionKey='0002'&Comments='test%20reject'"
                ]
                post_res = (grequests.post(u_post,
                                           auth=('pritamsa', 'rupu@0801'),
                                           headers=header_2,
                                           cookies=cookie)
                            for u_post in url_post)

                post_reque = grequests.map(post_res, size=1)
                response_array_post = []
                for response_post in post_reque:

                    if (response_post.status_code != 200):
                        print(
                            "hey problem in rejecting P.O. . Please try again later."
                        )
                        return rejection_failure_reply, bot_memo[
                            'index'], present_task_instance_id, bot_memo[
                                'created_by'], bot_memo['SupplierName'], bot_memo[
                                    'PurchaseOrderNetAmount'], rejection_failure_reply, '', '', bot_memo[
                                        'scrapped_po_no']

                    else:

                        return after_rejection_reply, bot_memo[
                            'index'], present_task_instance_id, bot_memo[
                                'created_by'], bot_memo['SupplierName'], bot_memo[
                                    'PurchaseOrderNetAmount'], after_rejection_reply, '', '', bot_memo[
                                        'scrapped_po_no']  #after this call the "next" task showing skill in bot

    # THIS LOGIC BELOW NEEDS TO BE RE_WRITTEN
    #************************************************************************************************************

    # elif((bot_nlp['ordinal'] and len(bot_nlp['ordinal']) <= bot_memo['no_of_line_items']) and present_skill == 'get_item_details'):
    elif (present_skill == 'get_item_details'):
        if (bot_nlp['ordinal']
                and len(bot_nlp['ordinal']) <= bot_memo['no_of_line_items']):
            # filter_item_ordinally = 'item : '+ (bot_nlp['ordinal'][bot_nlp['ordinal']['index']]['rank'])
            # print(filter_item_ordinally)
            print('///////////////////////////////////////////////////')
            nlp_ordinal_filter_index = bot_nlp['ordinal'][0][
                'index']  #this is the first element's index of nlp entity ordinal array
            individual_item_filter_string = 'item : ' + str(
                nlp_ordinal_filter_index + 1)
            item_level_reply_ordinally = bot_memo['all_item_details'][
                individual_item_filter_string]
            print(item_level_reply_ordinally)

            return str(item_level_reply_ordinally).strip(
                '{}'
            ), bot_memo['index'], bot_memo['instanceID'], bot_memo[
                'created_by'], bot_memo['SupplierName'], bot_memo[
                    'PurchaseOrderNetAmount'], bot_memo[
                        'after_approval_reply'], bot_memo[
                            'all_item_details'], bot_memo[
                                'no_of_line_items'], bot_memo['scrapped_po_no']

        elif (bot_nlp['ordinal'] == False and bot_nlp['number']
              and len(bot_nlp['number']) <= bot_memo['no_of_line_items']):
            # filter_item_ordinally = 'item : '+ (bot_nlp['ordinal'][bot_nlp['ordinal']['index']]['rank'])
            # print(filter_item_ordinally)
            print('///////////////////////////////////////////////////')
            nlp_number_filter_index = bot_nlp['number'][0][
                'scalar']  #this is the first element's index of nlp entity ordinal array
            individual_item_filter_string = 'item : ' + str(
                nlp_number_filter_index)
            item_level_reply_numerically = bot_memo['all_item_details'][
                individual_item_filter_string]
            print(item_level_reply_numerically)

            return str(
                item_level_reply_numerically
            ), bot_memo['index'], bot_memo['instanceID'], bot_memo[
                'created_by'], bot_memo['SupplierName'], bot_memo[
                    'PurchaseOrderNetAmount'], bot_memo[
                        'after_approval_reply'], bot_memo[
                            'all_item_details'], bot_memo[
                                'no_of_line_items'], bot_memo['scrapped_po_no']
Example #34
0
def convert(urls2, pf, pft, etfnames):
    # Start empty list of ETF holdings
    etf_lib = list()
    # Async requests
    reqs = [grequests.get(url) for url in urls2]
    resp = grequests.map(reqs)
    # Parse data tables with pandas and append data to return variable
    for i, r in enumerate(resp):
        get_text = r.text
        try:
            wds = pd.read_html(get_text)[1]
        # This statement skips equities so the scraper can pull ETF data only
        except:
            pass
        else:
            wds = wds.rename(columns={
                "%\xa0Weight": "% Weight",
                "%\xa0Change": "% Chg"
            })
            # Filter col of interest and convert '% Weight' col from str to float, format Symbol col
            for j in range(0, len(wds['% Weight'])):
                wds.at[j, '% Weight'] = wds.at[j, '% Weight'].replace("%", "")
                wds.at[j, 'Symbol'] = wds.at[j, 'Symbol'].replace("-", ".")
            wds['% Weight'] = wds['% Weight'].astype(float)
            # Delete unused data
            del wds['Price']
            del wds['% Chg']
            # Create MISC ticker which represents the % holding of the ETF not accouted for by top 25 holdings
            etft = 100 - wds['% Weight'].sum()
            new_row = {'Symbol': (pf.at[i, 'Symbol']), '% Weight': etft}
            wds = wds.append(new_row, ignore_index=True)
            # Multiply ETF ticker list by weight in portfolio
            wds['% Weight'] = wds['% Weight'] * pf.at[i, '% Weight']
            # Append to list of ETF data and remove ETF ticker from list and remove original ticker entry after it has been parsed
            etf_lib.append(wds)
            pf.drop([i], inplace=True)
    # Concatenate lists together and sum any repeat tickers in list
    df = pd.concat(etf_lib)
    pf['% Weight'] *= 100
    df = df.append(pf)
    # Save names as a dict object with symbols as keys and names as values
    names = dict(zip(df['Symbol'], df['Name']))
    # This command will combine repeat tickers and sum their values, but doing so deletes the Name col
    df = df.groupby(['Symbol'], as_index=False).sum()
    out = df.sort_values(by='% Weight', ascending=False)
    # Add empty Name col to out df and re-index
    out["Name"] = ['' for _ in range(len(out))]
    columnsTitles = ['Symbol', 'Name', '% Weight', 'Total Value']
    out = out.reindex(columns=columnsTitles)
    # Correct name column to values saved in names dictionary
    for i in range(0, len(out['Symbol'])):
        for j in names.keys():
            if str(j) == str(out.at[i, 'Symbol']):
                out.at[i, 'Name'] = names.get(j)
    # Re-add ETF Names
    for i in range(0, len(out['Symbol'])):
        for j in etfnames.keys():
            if str(j) in str(out.at[i, 'Symbol']):
                out.at[i,
                       'Name'] = ('Misc ' + str(etfnames.get(j)) + ' Holdings')
    # Update and format before print out
    val = pft * (out['% Weight'] / 100)
    out['Total Value'] = val
    out['Total Value'] = out['Total Value'].round(decimals=2)
    out['% Weight'] = out['% Weight'].round(decimals=3)
    # Return result
    return out
	def fetchRaceAndGenderFor(cls, people):
		peopleCopy = list(people)

		# Get the search page for each person
		urls = list()
		for person in peopleCopy:
			url = "http://search.nndb.com/search/?type=unspecified&query="
			url += "+".join(person.name.split())
			urls.append(url)
		requests = [grequests.get(url) for url in urls]
		responses = grequests.map(requests)

		# Get the profile pages from each search page
		urls = []
		notFoundIndices = set()
		for i in range(len(responses)):
			response = responses[i]
			if not response:
				notFoundIndices.add(i)
				continue

			name = peopleCopy[i].name.split()
			if len(name) == 0:
				notFoundIndices.add(i)
				continue

			searchTree = html.fromstring(response.content)
			condition = '[contains(.//text(), "%s")' % name[0]
			if len(name) > 1:
				condition += ' and contains(.//text(), "%s")' % name[1]
			condition += ']'

			personList = searchTree.xpath('//table')[3].xpath('./tr')
			if len(personList) <= 1:
				notFoundIndices.add(i)
				continue

			personList = personList[1].xpath('./td' + condition)

			if not personList or len(personList[0].xpath('.//a/@href')) == 0:
				notFoundIndices.add(i)
				continue

			urls.append(personList[0].xpath('.//a/@href')[0])

		# remove the people that we couldn't find info for
		for i in sorted(list(notFoundIndices), reverse=True):
			del peopleCopy[i]

		requests = [grequests.get(url) for url in urls]
		responses = grequests.map(requests)

		# Go through responses and scrape race and gender info
		for i in range(len(responses)):
			response = responses[i]
			person = peopleCopy[i]

			personTree = html.fromstring(response.content)

			raceStr = '//p/b[text()="Race or Ethnicity:"]'
			raceStr += '/following-sibling::text()'
			if len(personTree.xpath(raceStr)) > 0:
				person.race = personTree.xpath(raceStr)[0].strip()

			genderStr = '//p/b[text()="Gender:"]/following-sibling::text()'
			if len(personTree.xpath(genderStr)) > 0:
				person.gender = personTree.xpath(genderStr)[0].strip()
Example #36
0
def main(file, url, async, verbose):
    f = open(file, 'r')
    num_lines = file_len(file)
    start_time = time.time()
    count = 0

    if verbose:
        print "[+] Testing URL's ".format(num_lines)
urls = []
    for line in f:
        line = line.rstrip('\n')
        formated_url = url.format(line)
        urls.append(formated_url)

    if async:
        rs = (grequests.get(u) for u in urls)
        responses = grequests.map(rs)
        for response in responses:
            if response.status_code != 404:
                print response.url + " : " + str(response.status_code)
                count += 1
    else:
        for url in urls:
            r = requests.get(url, verify=False)
        if r.status_code != 404:
                print  "[*] " + r.url + " : " + str(r.status_code)
                count += 1
        else if response.status_code == '401' or == '403':
            print response.url + "Access Denied or unauthorized access" 
            count += 1
       
Example #37
0
        urls.append(b)


def my_exception_handler(req, e):
    print(req)
    print(e)
    print(dir(req))
    print(dir(e))


MAX_CONNECTIONS = 50  # Number of connections you want to limit it to
print(1111111)
pages = len(urls)
for i in range(1, pages + 1, MAX_CONNECTIONS):
    print("1 Waiting %s" % i)  # Optional, so you see something is done.
    rs = (grequests.get(u, timeout=100, verify=False)
          for u in urls[i:i + MAX_CONNECTIONS])
    a = list(rs)
    time.sleep(0.2)  # You can change this to whatever you see works better.
    results = grequests.map(
        a, exception_handler=my_exception_handler
    )  # The key here is to extend, not append, not insert.
    print("result1 : %s" % len(results))
    for x in results:
        if x:
            print(x.status_code)
            try:
                soup = BeautifulSoup(x.text, 'html.parser')
                for link in soup.find_all('a'):
                    if link.get('href'):
                        if link.get('href').find('/providers/profile/') != -1:
Example #38
0
if args.nofaces:
    sub_names.remove("faces")
if args.noflags:
    sub_names.remove("flags")
if args.nosublist:
    sub_names.remove("sublist")
if args.html:
    file_names.append("Ranking.html")
if args.css:
    file_names.append("Ranking.css")

print("Saving directories...")
rs = [
    grequests.get(url + "/" + dir_names[i] + "/",
                  verify=False,
                  stream=False,
                  session=sessions[i % NUM_SESSIONS])
    for i in range(len(dir_names))
]
dir_reqs = grequests.map(rs)

output = args.output
if output is None:
    output = list(dir_reqs[dir_names.index("contests")].json().keys())[0]

print("Copying static files...")
copy_tree("cmsranking", output, update=1)

for i in range(len(dir_names)):
    if dir_reqs[i] is not None:
        os.makedirs(output + "/" + dir_names[i], exist_ok=True)
Example #39
0
def getQueries():
    O0OO0O0O00O0OOO00 = ''
    OO0O0O0O00OO0000O = ''
    O0O00OOOOOOO0O000 = ''
    O0OOOOOOOO0OO0O0O = ''
    OO0OOOOO0O0O0OOOO = ''
    OO00000OO0OOO0OOO = ''
    OOOO0O0O00OOO0O00 = ''
    OO0O0O000O00O0O00 = ''
    OO000OOOO0O0000O0 = ''
    OOO0O0OOOO0O0O00O = ''
    OOO00O00OO0O00O0O = ''
    OO0O000O00OOOO0O0 = ''
    O0OOO00OOO00OO00O = ''
    O0OOO000000OOOOOO = ''
    OO00OOO000O00OO00 = ''
    O0OO0OOO0O0OO0000 = ''
    OO00O000OO000OO0O = ''
    O00O000OO000O00O0 = ''
    O000O0O0O0OO0O00O = ''
    with open('pairs.txt', 'rb') as (O0OO0OO0000O000OO):
        O0OO00OO000OO0O00 = OOOO00OOOO00O000O.load(O0OO0OO0000O000OO)
    O000OOO0OOOO0O0OO = 0
    for OO0O0O0O0O0O00O00 in O0OO00OO000OO0O00:
        if O000OOO0OOOO0O0OO <= 50:
            O0OO0O0O00O0OOO00 += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 50 and O000OOO0OOOO0O0OO <= 100:
            OO0O0O0O00OO0000O += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 100 and O000OOO0OOOO0O0OO <= 150:
            O0O00OOOOOOO0O000 += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 150 and O000OOO0OOOO0O0OO <= 200:
            O0OOOOOOOO0OO0O0O += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 200 and O000OOO0OOOO0O0OO <= 250:
            OO0OOOOO0O0O0OOOO += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 250 and O000OOO0OOOO0O0OO <= 300:
            OO00000OO0OOO0OOO += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 300 and O000OOO0OOOO0O0OO <= 350:
            OOOO0O0O00OOO0O00 += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 350 and O000OOO0OOOO0O0OO <= 400:
            OO0O0O000O00O0O00 += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 400 and O000OOO0OOOO0O0OO <= 450:
            OO000OOOO0O0000O0 += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 450 and O000OOO0OOOO0O0OO <= 500:
            OOO0O0OOOO0O0O00O += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 500 and O000OOO0OOOO0O0OO <= 550:
            OOO00O00OO0O00O0O += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 550 and O000OOO0OOOO0O0OO <= 600:
            OO0O000O00OOOO0O0 += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 600 and O000OOO0OOOO0O0OO <= 650:
            O0OOO00OOO00OO00O += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 650 and O000OOO0OOOO0O0OO <= 700:
            O0OOO000000OOOOOO += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 700 and O000OOO0OOOO0O0OO <= 750:
            OO00OOO000O00OO00 += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 750 and O000OOO0OOOO0O0OO <= 800:
            O0OO0OOO0O0OO0000 += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 800 and O000OOO0OOOO0O0OO <= 850:
            OO00O000OO000OO0O += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 850 and O000OOO0OOOO0O0OO <= 900:
            O00O000OO000O00O0 += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1
        if O000OOO0OOOO0O0OO > 900 and O000OOO0OOOO0O0OO <= 950:
            O000O0O0O0OO0O00O += str(OO0O0O0O0O0O00O00) + '-'
            O000OOO0OOOO0O0OO += 1

    querys.append(O0OO0O0O00O0OOO00[:-1])
    querys.append(OO0O0O0O00OO0000O[:-1])
    querys.append(O0O00OOOOOOO0O000[:-1])
    querys.append(O0OOOOOOOO0OO0O0O[:-1])
    querys.append(OO0OOOOO0O0O0OOOO[:-1])
    querys.append(OO00000OO0OOO0OOO[:-1])
    querys.append(OOOO0O0O00OOO0O00[:-1])
    querys.append(OO0O0O000O00O0O00[:-1])
    querys.append(OO000OOOO0O0000O0[:-1])
    querys.append(OOO0O0OOOO0O0O00O[:-1])
    querys.append(OOO00O00OO0O00O0O[:-1])
    querys.append(OO0O000O00OOOO0O0[:-1])
    querys.append(O0OOO00OOO00OO00O[:-1])
    querys.append(O0OOO000000OOOOOO[:-1])
    querys.append(OO00OOO000O00OO00[:-1])
    O0O000O0OO00000OO = 'https://yobit.net/api/3/ticker/'
    O0O0O00O000O0OOOO = (OOO0O00O0O000OO00.get(
        O0O000O0OO00000OO + OOO00OO0OO0O00OOO,
        headers={
            'apisign':
            OOO00OO000O000000.new(secret.encode(), OOO00OO0OO0O00OOO.encode(),
                                  OO0OOO000000OO000.sha512).hexdigest()
        }) for OOO00OO0OO0O00OOO in querys)
    OOO0OOO0OO000000O = OOO0O00O0O000OO00.map(O0O0O00O000O0OOOO)
    O0O0000O0O0OOOOOO = [
        O00OOO0O00OOOOO0O.json() for O00OOO0O00OOOOO0O in OOO0OOO0OO000000O
    ]
    for O000O000O0000O0OO in O0O0000O0O0OOOOOO:
        for OO0O00OOO00O00000 in O000O000O0000O0OO:
            O0O0000O0O00OO000 = OO0O00OOO00O00000
            volume.append(O000O000O0000O0OO[O0O0000O0O00OO000]['vol'])
            price.append(O000O000O0000O0OO[O0O0000O0O00OO000]['last'])
            names.append(O0O0000O0O00OO000)
Example #40
0
def main():
    global hashes
    hashes = set()
    past_urls = set()

    now = datetime.datetime.now()

    parser = argparse.ArgumentParser()
    parser.add_argument("-p",
                        "--proxy",
                        help="Define HTTP proxy as address:port")
    parser.add_argument("-d",
                        "--dumpdir",
                        help="Define dump directory for retrieved files")
    parser.add_argument("-l",
                        "--logfile",
                        help="Define file for logging progress")
    parser.add_argument("-x",
                        "--vxcage",
                        help="Dump the files to a VxCage instance",
                        action="store_true",
                        default=False)
    parser.add_argument("-v",
                        "--viper",
                        help="Dump the files to a Viper instance",
                        action="store_true",
                        default=False)
    parser.add_argument("-c",
                        "--cuckoo",
                        help="Enable Cuckoo analysis",
                        action="store_true",
                        default=False)
    parser.add_argument("-s",
                        "--sort_mime",
                        help="Sort files by MIME type",
                        action="store_true",
                        default=False)

    global cfg
    cfg = dict()
    args = parser.parse_args()

    global config
    config = ConfigParser.ConfigParser()
    config.read('maltrieve.cfg')

    if args.logfile or config.get('Maltrieve', 'logfile'):
        if args.logfile:
            cfg['logfile'] = args.logfile
        else:
            cfg['logfile'] = config.get('Maltrieve', 'logfile')
        logging.basicConfig(filename=cfg['logfile'],
                            level=logging.DEBUG,
                            format='%(asctime)s %(thread)d %(message)s',
                            datefmt='%Y-%m-%d %H:%M:%S')
    else:
        logging.basicConfig(level=logging.DEBUG,
                            format='%(asctime)s %(thread)d %(message)s',
                            datefmt='%Y-%m-%d %H:%M:%S')

    if args.proxy:
        cfg['proxy'] = {'http': args.proxy}
    elif config.has_option('Maltrieve', 'proxy'):
        cfg['proxy'] = {'http': config.get('Maltrieve', 'proxy')}
    else:
        cfg['proxy'] = None

    if config.has_option('Maltrieve', 'User-Agent'):
        cfg['User-Agent'] = {
            'User-Agent': config.get('Maltrieve', 'User-Agent')
        }
    else:
        cfg['User-Agent'] = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)"

    cfg['sort_mime'] = args.sort_mime

    if cfg['proxy']:
        logging.info('Using proxy %s', cfg['proxy'])
        my_ip = requests.get('http://ipinfo.io/ip', proxies=cfg['proxy']).text
        logging.info('External sites see %s', my_ip)
        print "External sites see %s" % my_ip

    cfg['vxcage'] = args.vxcage or config.has_option('Maltrieve', 'vxcage')
    cfg['cuckoo'] = args.cuckoo or config.has_option('Maltrieve', 'cuckoo')
    cfg['viper'] = args.viper or config.has_option('Maltrieve', 'viper')
    cfg['logheaders'] = config.get('Maltrieve', 'logheaders')

    black_list = []
    if config.has_option('Maltrieve', 'black_list'):
        black_list = config.get('Maltrieve', 'black_list').strip().split(',')

    white_list = False
    if config.has_option('Maltrieve', 'white_list'):
        white_list = config.get('Maltrieve', 'white_list').strip().split(',')

    # make sure we can open the directory for writing
    if args.dumpdir:
        cfg['dumpdir'] = args.dumpdir
    elif config.get('Maltrieve', 'dumpdir'):
        cfg['dumpdir'] = config.get('Maltrieve', 'dumpdir')
    else:
        cfg['dumpdir'] = '/tmp/malware'

    # Create the dir
    if not os.path.exists(cfg['dumpdir']):
        os.makedirs(cfg['dumpdir'])

    try:
        d = tempfile.mkdtemp(dir=cfg['dumpdir'])
    except Exception as e:
        logging.error('Could not open %s for writing (%s), using default',
                      cfg['dumpdir'], e)
        cfg['dumpdir'] = '/tmp/malware'
    else:
        os.rmdir(d)

    logging.info('Using %s as dump directory', cfg['dumpdir'])

    if os.path.exists('hashes.json'):
        with open('hashes.json', 'rb') as hashfile:
            hashes = json.load(hashfile)
    elif os.path.exists('hashes.obj'):
        with open('hashes.obj', 'rb') as hashfile:
            hashes = pickle.load(hashfile)

    if os.path.exists('urls.json'):
        with open('urls.json', 'rb') as urlfile:
            past_urls = json.load(urlfile)
    elif os.path.exists('urls.obj'):
        with open('urls.obj', 'rb') as urlfile:
            past_urls = pickle.load(urlfile)

    print "Processing source URLs"

    source_urls = {
        'https://zeustracker.abuse.ch/monitor.php?urlfeed=binaries':
        process_xml_list_desc,
        'http://www.malwaredomainlist.com/hostslist/mdl.xml':
        process_xml_list_desc,
        'http://malc0de.com/rss/': process_xml_list_desc,
        'http://vxvault.siri-urz.net/URL_List.php': process_simple_list,
        'http://urlquery.net/': process_urlquery,
        'http://support.clean-mx.de/clean-mx/rss?scope=viruses&limit=0%2C64':
        process_xml_list_title,
        'http://malwareurls.joxeankoret.com/normal.txt': process_simple_list
    }
    headers = {'User-Agent': 'Maltrieve'}

    reqs = [
        grequests.get(url, timeout=60, headers=headers, proxies=cfg['proxy'])
        for url in source_urls
    ]
    source_lists = grequests.map(reqs)

    print "Completed source processing"

    headers['User-Agent'] = cfg['User-Agent']
    malware_urls = set()
    for response in source_lists:
        if hasattr(response, 'status_code') and response.status_code == 200:
            malware_urls.update(source_urls[response.url](response.text))

    print "Downloading samples, check log for details"

    malware_urls -= past_urls
    reqs = [
        grequests.get(url, headers=headers, proxies=cfg['proxy'])
        for url in malware_urls
    ]
    for chunk in chunker(reqs, 32):
        malware_downloads = grequests.map(chunk)
        for each in malware_downloads:
            if not each or each.status_code != 200:
                continue
            md5 = save_malware(each, cfg['dumpdir'], black_list, white_list)
            if not md5:
                continue
            past_urls.add(each.url)

    print "Completed downloads"

    if past_urls:
        logging.info('Dumping past URLs to file')
        with open('urls.json', 'w') as urlfile:
            json.dump(past_urls, urlfile)

    if hashes:
        with open('hashes.json', 'w') as hashfile:
            json.dump(hashes, hashfile)
def request_cpu_load_from_nodes():
    # ******************************
    # Get Values from Hypervisors
    # ******************************

    global urls_hypervisors

    rs = (grequests.get(u, timeout=TIMEOUT) for u in urls_hypervisors)

    results = grequests.map(rs)
    if len(results) >= 0:
        if results[0] is not None:
            push_value_to_list(
                data_cpuload_time1,
                loads(results[0].content.decode('utf-8'))['total'])
        if results[1] is not None:
            push_value_to_list(
                data_cpuload_time2,
                loads(results[1].content.decode('utf-8'))['total'])
        if results[2] is not None:
            push_value_to_list(
                data_cpuload_time3,
                loads(results[2].content.decode('utf-8'))['total'])
        if results[3] is not None:
            push_value_to_list(
                data_cpuload_time4,
                loads(results[3].content.decode('utf-8'))['total'])

    # ******************************
    # Get Values from FortiGates
    # ******************************

    global fgt_sessions
    global urls_fgt

    fgt_login_requests = [None] * len(urls_fgt)
    fgt_cpu_requests = [None] * len(urls_fgt)

    # First, request CPU data

    for i in range(len(fgt_sessions)):
        fgt_cpu_requests[i] = grequests.get(
            urls_fgt[i] +
            'api/v2/monitor/system/resource/usage?resource=cpu&interval=1-min',
            session=fgt_sessions[i],
            headers=fgt_sessions[i].headers,
            timeout=TIMEOUT,
            verify=False)

    fgt_cpu_results = grequests.map(fgt_cpu_requests)

    # Check if request failed because of login
    # If failed, then login
    print("fgt_cpu_results:", fgt_cpu_results)

    reqs = []
    for i in range(len(fgt_sessions)):
        if fgt_cpu_results[i] is not None and fgt_cpu_results[
                i].status_code == 401:
            print("Login into FortiGate's REST API: ", i)
            fgt_login_requests[i] = grequests.post(
                urls_fgt[i] + 'logincheck',
                data='username='******'&secretkey=' + PASSWORD_FGT +
                '&ajax=1',
                session=fgt_sessions[i],
                timeout=TIMEOUT,
                verify=False)
            r = grequests.send(fgt_login_requests[i])
            reqs.append(r)
    gevent.joinall(reqs)

    # Only if request to get CPU was 200 OK then
    # get the value and push it to the list

    for i in range(len(fgt_cpu_results)):
        if fgt_cpu_results[i] and fgt_cpu_results[i].status_code == 200:
            try:
                push_value_to_list(
                    globals()['data_fgtload_time' + str(i + 1)],
                    loads(fgt_cpu_results[i].content.decode('utf-8'))
                    ['results']['cpu'][0]['current'])
            except:
                print("Error getting data from FortiGate:", i)
        else:
            print("FGT request was not ok:", i)
            if fgt_cpu_results[i] is not None:
                print("  -> result: ", fgt_cpu_results[i].status_code)
            push_value_to_list(globals()['data_fgtload_time' + str(i + 1)], -1)

    # ********************************
    # Get Values from DSO CyberMapper
    # ********************************

    global url_cybermapper

    # Get dpid

    loadbal = requests.get(url_cybermapper + '/v1.0/loadbal', timeout=TIMEOUT)

    # Use this notation '[*' to get the keys extracted into a list
    dpid = [*loads(loadbal.content.decode('utf-8')).keys()][0]

    # Get port statistics

    results = requests.get(url_cybermapper + '/v1.0/switch_stats/switches/' +
                           dpid + '/port_stats',
                           timeout=TIMEOUT)

    port_stats = loads(results.content.decode('utf-8'))

    bps = {}

    for port in port_stats:
        bps[port['id']] = (port['tx_bytes'] - port['last']['tx_bytes'] +
                           port['rx_bytes'] - port['last']['rx_bytes']) / \
                          (port['timestamp'] - port['last']['timestamp'])

    # Instead of port3 (which is faulty) we use 21-24
    # Instead of port4 we use 25-28
    push_value_to_list(data_totalthroughput_ingress_time,
                       (bps[1] + bps[21] + bps[22] + bps[23] + bps[24]) /
                       1000000000 * 8)
    push_value_to_list(data_totalthroughput_egress_time,
                       (bps[2] + bps[25] + bps[26] + bps[27] + bps[28]) /
                       1000000000 * 8)

    push_value_to_list(data_fgtthroughput1_time,
                       (bps[5] + bps[6]) / 2000000000 * 8)
    push_value_to_list(data_fgtthroughput2_time,
                       (bps[7] + bps[8]) / 2000000000 * 8)
    push_value_to_list(data_fgtthroughput3_time,
                       (bps[9] + bps[10]) / 2000000000 * 8)
    push_value_to_list(data_fgtthroughput4_time,
                       (bps[11] + bps[12]) / 2000000000 * 8)
    push_value_to_list(data_fgtthroughput5_time,
                       (bps[13] + bps[14]) / 2000000000 * 8)
    push_value_to_list(data_fgtthroughput6_time,
                       (bps[15] + bps[16]) / 2000000000 * 8)
def main():
    tstUrl = '/nekretnine/prodaja-savski-gaj-neadaptirana-kuca-150-m2-garaza-28m2-100.000-oglas-18694996'
    tstId = 18694996

    print 'start MAIN'
    global exportFeatures
    global oglasData

    #Params
    class Settings:
        None

    Settings.baseURL = 'http://www.njuskalo.hr'
    Settings.startURL = 'http://www.njuskalo.hr/prodaja-kuca/zagreb'
    Settings.startURL = 'http://www.njuskalo.hr/prodaja-kuca/zagreb?page=%i'

    addRequestss = []

    pageRequestss = (grequests.get(Settings.startURL % pageNum)
                     for pageNum in range(1, 3))  # 1-156
    #print pageRequestss
    pageResponses = grequests.map(pageRequestss)

    #print pageResponses
    #print 'start pageResponse LOOP'
    for pageResponse in pageResponses:
        tree = html.fromstring(pageResponse.content)

        # get oglasi
        oglasi = tree.xpath(
            '//*[@id="form_browse_detailed_search"]/div/div[1]/div[2]/div[4]/ul/li[@class!="EntityList-item--banner"]'
        )

        #//*[@id="form_browse_detailed_search"]/div/div[1]/div[2]/div[4]/ul/li[1]/article/h3/a

        for oglas in oglasi:
            dataAddId = oglas.xpath('@data-ad-id')
            #print dataAddId
            if dataAddId:
                dataAddId = [tstId]
                #dataAddId = dataAddId[0]
                addTitle = oglas.xpath('article/h3/a/text()')
                #print addTitle
                addUrl = tstUrl  #oglas.xpath('article/h3/a/@href')[0]
                #print addUrl

                #print Settings.baseURL + addUrl

                oglasData[(Settings.baseURL + addUrl)] = {
                    'dataAddId': dataAddId[0],
                    'addTitle': addTitle[0],
                    'addUrl': (Settings.baseURL + addUrl)
                }

                addRequestss.append(
                    grequests.get(Settings.baseURL + addUrl,
                                  hooks={'response': addProcess}))

        #print 'END pageResponse LOOP %s' % pageResponse

    # get individual adds
    #print addRequestss
    addResponses = grequests.map(addRequestss)

    #print addResponses

    featuresGeoJson = FeatureCollection(exportFeatures)

    f = open('data_dump_20161022.geojson', 'w')
    f.write(str(featuresGeoJson))  # python will convert \n to os.linesep
    f.close()
Example #43
0
def test_query_partners_by_token(
        api_backend,
        api_test_context,
        api_raiden_service):
    # let's create 2 new channels for the same token
    first_partner_address = '0x61c808d82a3ac53231750dadc13c777b59310bd9'
    second_partner_address = '0x29fa6cf0cce24582a9b20db94be4b6e017896038'
    token_address = '0xea674fdde714fd979de3edf0f56aa9716b898ec8'
    settle_timeout = 1650
    channel_data_obj = {
        'partner_address': first_partner_address,
        'token_address': token_address,
        'settle_timeout': settle_timeout
    }
    request = grequests.put(
        api_url_for(api_backend, 'channelsresource'),
        json=channel_data_obj
    )
    response = request.send().response
    assert_proper_response(response)
    response = response.json()
    first_channel_address = response['channel_address']

    channel_data_obj['partner_address'] = second_partner_address
    request = grequests.put(
        api_url_for(api_backend, 'channelsresource'),
        json=channel_data_obj,
    )
    response = request.send().response
    assert_proper_response(response)
    response = response.json()
    second_channel_address = response['channel_address']

    # and a channel for another token
    channel_data_obj['partner_address'] = '0xb07937AbA15304FBBB0Bf6454a9377a76E3dD39E'
    channel_data_obj['token_address'] = '0x70faa28A6B8d6829a4b1E649d26eC9a2a39ba413'
    request = grequests.put(
        api_url_for(api_backend, 'channelsresource'),
        json=channel_data_obj
    )
    response = request.send().response
    assert_proper_response(response)

    # and now let's query our partners per token for the first token
    request = grequests.get(
        api_url_for(
            api_backend,
            'partnersresourcebytokenaddress',
            token_address=token_address,
        )
    )
    response = request.send().response
    assert_proper_response(response)
    response = response.json()
    expected_response = [
        {
            'partner_address': first_partner_address,
            'channel': '/api/1/channels/{}'.format(first_channel_address)
        }, {
            'partner_address': second_partner_address,
            'channel': '/api/1/channels/{}'.format(second_channel_address)
        }
    ]
    assert all(r in response for r in expected_response)
Example #44
0
def test_connect_and_leave_token_network(
        api_backend,
        api_test_context,
        api_raiden_service):

    # first check we don't have any open channels
    request = grequests.get(
        api_url_for(api_backend, 'channelsresource')
    )
    response = request.send().response
    assert_proper_response(response)
    channels = response.json()
    assert not channels
    assert response.json() == api_test_context.expect_channels()

    funds = 100
    initial_channel_target = DEFAULT_INITIAL_CHANNEL_TARGET
    joinable_funds_target = DEFAULT_JOINABLE_FUNDS_TARGET
    token_address = '0xea674fdde714fd979de3edf0f56aa9716b898ec8'
    connect_data_obj = {
        'funds': funds,
    }
    request = grequests.put(
        api_url_for(api_backend, 'connectionsresource', token_address=token_address),
        json=connect_data_obj,
    )
    response = request.send().response
    assert_proper_response(response)

    # check that channels got created
    request = grequests.get(
        api_url_for(api_backend, 'channelsresource')
    )
    response = request.send().response
    assert_proper_response(response)
    channels = response.json()
    # There should be three channels according to the default initial_channel_target
    assert len(channels) == DEFAULT_INITIAL_CHANNEL_TARGET
    assert response.json() == api_test_context.expect_channels()

    expected_balance = int((funds * joinable_funds_target) / initial_channel_target)
    assert channels[0]['balance'] == expected_balance
    assert channels[1]['balance'] == expected_balance
    assert channels[2]['balance'] == expected_balance
    assert channels[0]['state'] == CHANNEL_STATE_OPENED
    assert channels[1]['state'] == CHANNEL_STATE_OPENED
    assert channels[2]['state'] == CHANNEL_STATE_OPENED

    # Let's leave the token network
    request = grequests.delete(
        api_url_for(api_backend, 'connectionsresource', token_address=token_address),
    )
    response = request.send().response
    assert_proper_response(response)

    # check that all channels were settled after calling `leave`
    request = grequests.get(
        api_url_for(api_backend, 'channelsresource')
    )
    response = request.send().response
    assert_proper_response(response)

    channels = response.json()
    assert channels[0]['state'] == CHANNEL_STATE_SETTLED
    assert channels[1]['state'] == CHANNEL_STATE_SETTLED
    assert channels[2]['state'] == CHANNEL_STATE_SETTLED
Example #45
0
    for i in range(1, 589):
        L.append("https://supreme.justia.com/cases/federal/us/" + str(i))

    while (len(L) > 0):
        print(len(L))
        ctr = 0
        curr_urls = []
        while (ctr < 100 and len(L) > 0):
            curr = L.popleft()
            if not seen.__contains__(curr):
                ctr += 1
                seen.add(curr)
                curr_urls.append(curr)

        indices = [url for i, url in enumerate(curr_urls)]
        rs = [grequests.get(u, timeout=3) for u in curr_urls]
        grequests.map(rs)
        for r in rs:
            resp = r.response
            if resp is None or resp.status_code is None or resp.status_code != 200:
                print(f"REQUEUING {indices[i]}")
                L.append(indices[i])
                seen.remove(indices[i])
                continue
            resp_url = r.response.url
            stripped_resp_url = strip_url(resp_url)
            url_ct += 1
            data = r.response.text
            soup = BeautifulSoup(data, "lxml")
            for link in soup.find_all('a'):
                if link.has_attr('href'):
Example #46
0
 def async_request(urls, err_callback=None, pool_size=5, *args, **kw):
     import grequests
     tasks = [grequests.get(u, **kw) for u in urls]
     return grequests.map(tasks, size=pool_size, exception_handler=err_callback)
Example #47
0
def test_api_open_and_deposit_channel(
        api_backend,
        api_test_context,
        api_raiden_service):
    # let's create a new channel
    first_partner_address = '0x61c808d82a3ac53231750dadc13c777b59310bd9'
    token_address = '0xea674fdde714fd979de3edf0f56aa9716b898ec8'
    settle_timeout = 1650
    channel_data_obj = {
        'partner_address': first_partner_address,
        'token_address': token_address,
        'settle_timeout': settle_timeout
    }
    request = grequests.put(
        api_url_for(api_backend, 'channelsresource'),
        json=channel_data_obj
    )
    response = request.send().response

    assert_proper_response(response)
    response = response.json()
    expected_response = channel_data_obj
    expected_response['balance'] = 0
    expected_response['state'] = CHANNEL_STATE_OPENED
    # can't know the channel address beforehand but make sure we get one
    assert 'channel_address' in response
    first_channel_address = response['channel_address']
    expected_response['channel_address'] = response['channel_address']
    assert response == expected_response

    # now let's open a channel and make a deposit too
    second_partner_address = '0x29fa6cf0cce24582a9b20db94be4b6e017896038'
    balance = 100
    channel_data_obj = {
        'partner_address': second_partner_address,
        'token_address': token_address,
        'settle_timeout': settle_timeout,
        'balance': balance
    }
    request = grequests.put(
        api_url_for(api_backend, 'channelsresource'),
        json=channel_data_obj
    )
    response = request.send().response

    assert_proper_response(response)
    response = response.json()
    expected_response = channel_data_obj
    expected_response['balance'] = balance
    expected_response['state'] = CHANNEL_STATE_OPENED
    # can't know the channel address beforehand but make sure we get one
    assert 'channel_address' in response
    expected_response['channel_address'] = response['channel_address']
    second_channel_address = response['channel_address']
    assert response == expected_response

    # let's deposit on the first channel
    request = grequests.patch(
        api_url_for(
            api_backend,
            'channelsresourcebychanneladdress',
            channel_address=first_channel_address
        ),
        json={'balance': balance}
    )
    response = request.send().response
    assert_proper_response(response)
    response = response.json()
    expected_response = {
        'channel_address': first_channel_address,
        'partner_address': first_partner_address,
        'token_address': token_address,
        'settle_timeout': settle_timeout,
        'state': CHANNEL_STATE_OPENED,
        'balance': balance
    }
    assert response == expected_response

    # finally let's try querying for the second channel
    request = grequests.get(
        api_url_for(
            api_backend,
            'channelsresourcebychanneladdress',
            channel_address=second_channel_address
        )
    )

    response = request.send().response
    assert_proper_response(response)
    response = response.json()
    expected_response = {
        'channel_address': second_channel_address,
        'partner_address': second_partner_address,
        'token_address': token_address,
        'settle_timeout': settle_timeout,
        'state': CHANNEL_STATE_OPENED,
        'balance': balance
    }
    assert response == expected_response
Example #48
0
def run():
    stocks = []
    stock_map = {}
    df = pd.read_csv(
        '{data_path}/for_lives.csv'.format(data_path=env.data_path))
    for i in range(len(df)):
        ts_code = df.loc[i, 'ts_code']
        if ts_code.startswith('688'):
            continue

        market = ts_code.split('.')[1].lower()
        symbol = ts_code.split('.')[0]
        name = df.loc[i, 'name']
        stocks.append({'name': name, 'symbol': symbol, 'market': market})
        stock_map['{market}{symbol}'.format(market=market, symbol=symbol)] = {
            'local_max':
            float(df.loc[i, 'COL_DAILY_LOCAL_MAX']),
            'accumulation_aggressive':
            float(df.loc[i, 'COL_ACCUMULATION_AGGRESSIVE'])
        }

    # for i in open('{data_path}/silent_ones'.format(data_path=env.data_path)):
    #     infos = i.strip().split(',')
    #     if infos[1].startswith('688'):
    #         continue
    #     stocks.append({
    #         'name': infos[0],
    #         'symbol': infos[1],
    #         'market': infos[2]
    #     })
    #     stock_map['{market}{symbol}'.format(market=infos[2], symbol=infos[1])] = {
    #         'local_max': float(infos[3])
    #     }

    big_hands = {}
    time_str = time.strftime("%Y-%m-%d", time.localtime())
    pickle_file_name = '{buffer_path}/{time_str}@big_hand.pkl'.format(
        buffer_path=env.buffer_path, time_str=time_str)
    if os.path.exists(pickle_file_name):
        with open(pickle_file_name, 'rb') as file:
            try:
                big_hands = pickle.load(file)
            except EOFError:
                big_hands = {}

    batch_size = 500
    req_list = []
    for i in range(0, len(stocks), batch_size):
        keys = []
        for item in stocks[i:i + batch_size]:
            query_key = '{market}{symbol}'.format(market=item['market'],
                                                  symbol=item['symbol'])
            keys.append(query_key)

        req_list.append(
            grequests.get('http://hq.sinajs.cn/list={}'.format(
                ','.join(keys))))

    while True:
        time_a = time.time()
        try:
            responses = grequests.map(req_list)
            print('====== {} ======'.format(
                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
            local_breaks = []
            for response in responses:
                res = response.text.strip().split(';\n')
                for i in res:
                    j = i.split(',')
                    name = j[0].split('="')[1]
                    code = j[0].split('="')[0].split('_')[-1]
                    yesterday_closing_price = float(j[2])
                    current_price = float(j[3])
                    chg = (current_price / yesterday_closing_price - 1)
                    chg_display = '{}%'.format(round(chg * 100, 2))
                    local_max = stock_map[code]['local_max']
                    vols = int(j[8]) / 100
                    # update
                    if 'vols' in stock_map[code]:
                        last_vols = stock_map[code]['vols']
                        delta_vols = vols - last_vols
                    else:
                        # first time
                        stock_map[code]['vols'] = vols
                        delta_vols = 0

                    if delta_vols >= 10000:
                        if code not in big_hands:
                            big_hands[code] = {'max_delta_vols': delta_vols}
                        if delta_vols > big_hands[code]['max_delta_vols']:
                            big_hands[code]['max_delta_vols'] = delta_vols

                            with open(pickle_file_name, 'wb') as file:
                                pickle.dump(big_hands, file)

                    if current_price > local_max and 0.05 < chg < 0.098 and code in big_hands:
                        local_breaks.append({
                            'note':
                            'local break\t{code}\t{name}\tchg:{chg}\tprice:{price}\tbig_hands:{big_hands}\t\taccumulation:{accumulation}'
                            .format(
                                code=code,
                                name=name,
                                chg=chg_display,
                                price=round(current_price, 2),
                                big_hands=int(
                                    big_hands[code]['max_delta_vols']),
                                accumulation=round(
                                    stock_map[code]['accumulation_aggressive'],
                                    2)),
                            'chg':
                            chg
                        })
                    #
                    # if 0.05 < chg < 0.098:
                    #     if current_price > local_max:
                    #         local_breaks.append({
                    #             'note': 'local break\t{code}\t{name}\tchg:{chg}\tprice:{price}'.format(code=code, name=name, chg=chg_display, price=round(current_price, 2)),
                    #             'chg': chg
                    #         })
                    #     else:
                    #         # print('violent move\t{code}\t{name}\tchg:{chg}\tprice:{price}'.format(code=code, name=name, chg=chg_display, price=round(current_price, 2)))
                    #         pass
            local_breaks.sort(key=lambda x: x['chg'], reverse=True)
            notes = [i['note'] for i in local_breaks]
            print('\n'.join(notes))
        except Exception as e:
            print(e)
            continue
        time_b = time.time()
        cost = time_b - time_a
        time.sleep(1 - cost)
Example #49
0
#!/usr/bin/python
# -*- coding: utf-8 -*-
#author:闻到 http://wendao123.cn
#抓取斗图啦http://www.doutula.com的表情包,并按文件夹存储

import requests
from bs4 import BeautifulSoup
import os, re, grequests
# import hashlib
url = 'http://www.doutula.com/article/list/?page={page}'

#请求队列,还未发出请求
urls = (grequests.get(url.format(page=page)) for page in range(1, 544))

#批量发出请求,得到响应的列表resps
response = grequests.map(urls)

for res in response:
    doc = BeautifulSoup(res.text, 'lxml')
    list_group_item = doc.find_all('a', class_='list-group-item random_list')
    for group_item in list_group_item:
        title = group_item.find(class_='random_title').text
        # print(title[:-10])
        img_urls = group_item.find_all(class_='lazy image_dtb img-responsive')
        # print(img_urls)
        for img_url in img_urls:
            alt = img_url.get('alt')
            true_img_url = img_url.get('data-backup')[:-4]
            hz = true_img_url[-4:]
            # print(alt,true_img_url,hz)
            img_response = requests.get(true_img_url)
Example #50
0
def test_payment_events_endpoints(api_backend, raiden_network,
                                  token_addresses):
    _, app1, app2 = raiden_network
    amount = 200
    identifier = 42
    token_address = token_addresses[0]

    target1_address = app1.raiden.address
    target2_address = app2.raiden.address

    api_server, _ = api_backend

    # sending tokens to target 1
    request = grequests.post(
        api_url_for(
            api_backend,
            'token_target_paymentresource',
            token_address=to_checksum_address(token_address),
            target_address=to_checksum_address(target1_address),
        ),
        json={
            'amount': amount,
            'identifier': identifier
        },
    )
    request.send()

    # sending some tokens to target 2
    amount -= 10
    request = grequests.post(
        api_url_for(
            api_backend,
            'token_target_paymentresource',
            token_address=to_checksum_address(token_address),
            target_address=to_checksum_address(target2_address),
        ),
        json={
            'amount': amount,
            'identifier': identifier
        },
    )
    request.send()

    # test endpoint without (partner and token)
    request = grequests.get(api_url_for(
        api_backend,
        'paymentresource',
    ), )
    response = request.send().response
    assert_proper_response(response, HTTPStatus.OK)
    response = response.json()
    assert len(response) == 2
    assert response[0]['event'] == 'EventPaymentSentSuccess'
    assert response[1]['event'] == 'EventPaymentSentSuccess'

    # test endpoint without partner
    request = grequests.get(
        api_url_for(
            api_backend,
            'token_paymentresource',
            token_address=token_address,
        ), )
    response = request.send().response
    assert_proper_response(response, HTTPStatus.OK)
    response = response.json()
    assert len(response) == 2
    assert response[0]['event'] == 'EventPaymentSentSuccess'
    assert response[1]['event'] == 'EventPaymentSentSuccess'

    # test endpoint for token and partner
    request = grequests.get(
        api_url_for(
            api_backend,
            'token_target_paymentresource',
            token_address=token_address,
            target_address=target1_address,
        ), )
    response = request.send().response
    assert_proper_response(response, HTTPStatus.OK)
    response = response.json()
    assert len(response) == 1
    assert response[0]['event'] == 'EventPaymentSentSuccess'
    "https://www.youtube.com/user/{}/about?hl=en".format(handle),
    "https://twitter.com/{}".format(handle),
    "https://www.instagram.com/{}".format(handle),
    "https://www.facebook.com/{}".format(handle)
]

font = ImageFont.truetype("OpenSans-Italic.ttf", 30)
tcolor = (244, 210, 77)
text_pos1 = (50, 10)
text_pos2 = (620, 10)
text_pos3 = (50, 100)
text_pos4 = (620, 100)
img = Image.open("templates.jpg")

while True:
    rs = (grequests.get(u) for u in link_list)
    y, t, i, f = list(map(lambda r: r.text, grequests.map(rs)))
    ys = fetch_youtube_subscriber(y)
    tf = fetch_twitter_followers(t)
    instagram_followers = fetch_instagram_followers(i)
    ff = fetch_facebook_followers(f)

    draw = ImageDraw.Draw(img)
    draw.text(text_pos1, "  {} \n YouTube".format(ys), fill=tcolor, font=font)
    draw.text(text_pos2, "   {} \n Twitter".format(tf), fill=tcolor, font=font)
    draw.text(text_pos3,
              "     {} \n Instagram".format(instagram_followers),
              fill=tcolor,
              font=font)
    draw.text(text_pos4,
              "     {} \n Facebook".format(ff),
Example #52
0
def test_api_open_and_deposit_channel(
    api_backend,
    token_addresses,
    reveal_timeout,
):
    # let's create a new channel
    first_partner_address = '0x61C808D82A3Ac53231750daDc13c777b59310bD9'
    token_address = token_addresses[0]
    settle_timeout = 1650
    channel_data_obj = {
        'partner_address': first_partner_address,
        'token_address': to_checksum_address(token_address),
        'settle_timeout': settle_timeout,
        'reveal_timeout': reveal_timeout,
    }

    request = grequests.put(
        api_url_for(
            api_backend,
            'channelsresource',
        ),
        json=channel_data_obj,
    )
    response = request.send().response

    assert_proper_response(response, HTTPStatus.CREATED)
    response = response.json()
    expected_response = channel_data_obj
    expected_response['balance'] = 0
    expected_response['state'] = CHANNEL_STATE_OPENED
    expected_response[
        'channel_identifier'] = assert_dicts_are_equal.IGNORE_VALUE
    expected_response[
        'token_network_identifier'] = assert_dicts_are_equal.IGNORE_VALUE
    assert_dicts_are_equal(response, expected_response)

    token_network_identifier = response['token_network_identifier']

    # now let's open a channel and make a deposit too
    second_partner_address = '0x29FA6cf0Cce24582a9B20DB94Be4B6E017896038'
    balance = 100
    channel_data_obj = {
        'partner_address': second_partner_address,
        'token_address': to_checksum_address(token_address),
        'settle_timeout': settle_timeout,
        'reveal_timeout': reveal_timeout,
        'balance': balance,
    }
    request = grequests.put(
        api_url_for(
            api_backend,
            'channelsresource',
        ),
        json=channel_data_obj,
    )
    response = request.send().response

    assert_proper_response(response, HTTPStatus.CREATED)
    response = response.json()
    expected_response = channel_data_obj
    expected_response['balance'] = balance
    expected_response['state'] = CHANNEL_STATE_OPENED
    expected_response[
        'channel_identifier'] = assert_dicts_are_equal.IGNORE_VALUE
    expected_response['token_network_identifier'] = token_network_identifier
    assert_dicts_are_equal(response, expected_response)

    # assert depositing negative amount fails
    request = grequests.patch(
        api_url_for(
            api_backend,
            'channelsresourcebytokenandpartneraddress',
            token_address=token_address,
            partner_address=first_partner_address,
        ),
        json={'total_deposit': -1000},
    )
    response = request.send().response
    assert_proper_response(response, HTTPStatus.CONFLICT)
    # let's deposit on the first channel
    request = grequests.patch(
        api_url_for(
            api_backend,
            'channelsresourcebytokenandpartneraddress',
            token_address=token_address,
            partner_address=first_partner_address,
        ),
        json={'total_deposit': balance},
    )
    response = request.send().response
    assert_proper_response(response)
    response = response.json()
    expected_response = {
        'channel_identifier': assert_dicts_are_equal.IGNORE_VALUE,
        'partner_address': first_partner_address,
        'token_address': to_checksum_address(token_address),
        'settle_timeout': settle_timeout,
        'reveal_timeout': reveal_timeout,
        'state': CHANNEL_STATE_OPENED,
        'balance': balance,
        'token_network_identifier': token_network_identifier,
    }
    assert_dicts_are_equal(response, expected_response)

    # let's try querying for the second channel
    request = grequests.get(
        api_url_for(
            api_backend,
            'channelsresourcebytokenandpartneraddress',
            token_address=token_address,
            partner_address=second_partner_address,
        ), )

    response = request.send().response
    assert_proper_response(response)
    response = response.json()
    expected_response = {
        'channel_identifier': assert_dicts_are_equal.IGNORE_VALUE,
        'partner_address': second_partner_address,
        'token_address': to_checksum_address(token_address),
        'settle_timeout': settle_timeout,
        'reveal_timeout': reveal_timeout,
        'state': CHANNEL_STATE_OPENED,
        'balance': balance,
        'token_network_identifier': token_network_identifier,
    }
    assert_dicts_are_equal(response, expected_response)

    # finally let's burn all eth and try to open another channel
    api_server, _ = api_backend
    burn_all_eth(api_server.rest_api.raiden_api.raiden)
    channel_data_obj = {
        'partner_address': '0xf3AF96F89b3d7CdcBE0C083690A28185Feb0b3CE',
        'token_address': to_checksum_address(token_address),
        'settle_timeout': settle_timeout,
        'reveal_timeout': reveal_timeout,
        'balance': 1,
    }
    request = grequests.put(
        api_url_for(
            api_backend,
            'channelsresource',
        ),
        json=channel_data_obj,
    )
    response = request.send().response
    assert_proper_response(response, HTTPStatus.PAYMENT_REQUIRED)
    response = response.json()
    assert 'The account balance is below the estimated amount' in response[
        'errors']
Example #53
0
def muti_scrawl_page(urls):
    rs = (grequests.get(u) for u in urls)
    tmp = grequests.map(rs, size=10)
    return tmp
Example #54
0
def test_query_partners_by_token(api_backend, blockchain_services,
                                 token_addresses):
    first_partner_address = '0x61C808D82A3Ac53231750daDc13c777b59310bD9'
    second_partner_address = '0x29FA6cf0Cce24582a9B20DB94Be4B6E017896038'
    token_address = token_addresses[0]
    settle_timeout = 1650
    channel_data_obj = {
        'partner_address': first_partner_address,
        'token_address': to_checksum_address(token_address),
        'settle_timeout': settle_timeout,
    }
    request = grequests.put(
        api_url_for(
            api_backend,
            'channelsresource',
        ),
        json=channel_data_obj,
    )
    response = request.send().response
    assert_proper_response(response, HTTPStatus.CREATED)
    response = response.json()

    channel_data_obj['partner_address'] = second_partner_address
    request = grequests.put(
        api_url_for(
            api_backend,
            'channelsresource',
        ),
        json=channel_data_obj,
    )
    response = request.send().response
    assert_proper_response(response, HTTPStatus.CREATED)
    response = response.json()

    # and a channel for another token
    channel_data_obj[
        'partner_address'] = '0xb07937AbA15304FBBB0Bf6454a9377a76E3dD39E'
    channel_data_obj['token_address'] = to_checksum_address(token_address)
    request = grequests.put(
        api_url_for(
            api_backend,
            'channelsresource',
        ),
        json=channel_data_obj,
    )
    response = request.send().response
    assert_proper_response(response, HTTPStatus.CREATED)

    # and now let's query our partners per token for the first token
    request = grequests.get(
        api_url_for(
            api_backend,
            'partnersresourcebytokenaddress',
            token_address=to_checksum_address(token_address),
        ), )
    response = request.send().response
    assert_proper_response(response)
    response = response.json()
    expected_response = [
        {
            'partner_address':
            first_partner_address,
            'channel':
            '/api/1/channels/{}/{}'.format(
                to_checksum_address(token_address),
                to_checksum_address(first_partner_address),
            ),
        },
        {
            'partner_address':
            second_partner_address,
            'channel':
            '/api/1/channels/{}/{}'.format(
                to_checksum_address(token_address),
                to_checksum_address(second_partner_address),
            ),
        },
    ]
    assert all(r in response for r in expected_response)
Example #55
0
from itertools import permutations
import grequests

s = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
s = list(s)
base = 'https://pokemongo.gamepress.gg/sites/pokemongo/files/styles/240w/public/flork-images/'
for i in range(1, 5):
    print(i)
    rs = (grequests.get(f'{base}{"".join([str(y) for y in x])}.png')
          for x in permutations(s, i))
    requests = grequests.map(rs)
    for response in requests:
        if response:
            print(response.url)

urls = [
    'http://www.heroku.com', 'http://python-tablib.org', 'http://httpbin.org',
    'http://python-requests.org', 'http://kennethreitz.com'
]

rs = (grequests.get(u) for u in urls)
requests = grequests.map(rs)
for response in requests:
    print(response)
Example #56
0
def test_get_connection_managers_info(api_backend, token_addresses):
    # check that there are no registered tokens
    request = grequests.get(
        api_url_for(api_backend, 'connectionsinforesource'), )
    response = request.send().response
    result = response.json()
    assert len(result) == 0

    funds = 100
    token_address1 = to_checksum_address(token_addresses[0])
    connect_data_obj = {
        'funds': funds,
    }
    request = grequests.put(
        api_url_for(
            api_backend,
            'connectionsresource',
            token_address=token_address1,
        ),
        json=connect_data_obj,
    )
    response = request.send().response
    assert_no_content_response(response)

    # check that there now is one registered channel manager
    request = grequests.get(
        api_url_for(api_backend, 'connectionsinforesource'), )
    response = request.send().response
    result = response.json()
    assert isinstance(result, dict) and len(result.keys()) == 1
    assert token_address1 in result
    assert isinstance(result[token_address1], dict)
    assert set(result[token_address1].keys()) == {
        'funds', 'sum_deposits', 'channels'
    }

    funds = 100
    token_address2 = to_checksum_address(token_addresses[1])
    connect_data_obj = {
        'funds': funds,
    }
    request = grequests.put(
        api_url_for(
            api_backend,
            'connectionsresource',
            token_address=token_address2,
        ),
        json=connect_data_obj,
    )
    response = request.send().response
    assert_no_content_response(response)

    # check that there now are two registered channel managers
    request = grequests.get(
        api_url_for(api_backend, 'connectionsinforesource'), )
    response = request.send().response
    result = response.json()
    assert isinstance(result, dict) and len(result.keys()) == 2
    assert token_address2 in result
    assert isinstance(result[token_address2], dict)
    assert set(result[token_address2].keys()) == {
        'funds', 'sum_deposits', 'channels'
    }
Example #57
0
# first page
visited = set('qds.html')
stack = [requests.get(root + 'qds.html').text]

phrases = []
while len(stack) != 0:
    html = stack.pop()
    soup = BeautifulSoup(html, 'html.parser')

    # get phrase
    if soup.font:
        phrase = soup.font.contents[1]
        phrases.append(phrase)
        print(phrase)

    # get more links
    links = []
    for link in soup.find_all('a'):
        link = link.get('href')

        if link in visited:
            continue

        visited.add(link)
        links.append(grequests.get(root + link))
        print(link)

    # add all links responses to stack
    [stack.append(link.text) for link in grequests.map(links)]

[print(phrase) for phrase in sorted(phrases)]
Example #58
0
def main():

    # globals
    global concurrent # how many concurrent requests to make at once
    global host       # collection of all Page objects (see class Page), starting w/ passed target URL
    global dictionary # loaded words
    global quiet      # shh
    global schema     # webpage request schema (http:// or https://)

    # collect options and arguments
    try:
        options, a = getopt.getopt(sys.argv[1:], "c:halto:q", [
            "conc=",
            "help",
            "all",
            "listable",
            "tree",
            "out=",
            "quiet"
        ])
    except getopt.GetoptError as err:
        print(str(err))
        usage()
        sys.exit(1)

    # init flag / config variables
    man_args = a # program takes mandatory positional args, so anything not option-flagged goes here.
    all_flag = False
    list_flag = False
    tree_flag = False
    out_file = None
    concurrent = 15
    quiet = False

    # now handle our options
    for option, argument in options:
        if option in ("-c", "--conc"):
            try:
                concurrent = int(argument)
            except ValueError:
                print("We both know %s needs to be passed a number. Don't be a s******d.\n" % option)
                usage()
                sys.exit(1)

        elif option in ("-h", "--help"):
            usage()
            sys.exit(0)

        elif option in ("-a", "--all"):
            all_flag = True

        elif option in ("-t", "--tree"):
            tree_flag = True

        elif option in ("-l", "--listable"):
            list_flag = True

        elif option in ("-o", "--out"):
            out_file = argument

        elif option in ("-q", "--quiet"):
            quiet = True

        else:
            assert False, "unhandled option"
    
    # handle mandatory / positional arguments
    if len(man_args) < 2:
        print("Error: Missing %d of 2 mandatory arguments!" % (2 - len(man_args)))
        print("  required syntax: slasher.py <host> <dictionary>")
        print("  run with -h or --help for usage")
        sys.exit(1)

    elif len(man_args) >= 3:
        print("[!] Unrecognized positional arguments:")
        print("    " + str(man_args[2:]) + " will not be used!")

        while True:
            r = input("    Continue? Y/N\n    > ")
            if r in "yY":
                print("[*] Proceeding, gnoring unrecognized args...")
                break

            elif r in "nN":
                print("[!] User chose to end scan.")
                sys.exit(1)

            print("\n[!] Invalid response. Please try again.")

    # setup our starting page and get path to our wordlist
    host = [ [Page(man_args[0], None, False, True)] ] # 2d list. ctrl+f "main scan loop" for deets on why it's this way
    top = host[0][0] # this is just for convenience when we need to reference only the main site page.
    list_path = man_args[1]

    # get schema
    if "https://" in top.uri:
        schema = "https://"
        top.uri = top.uri[8:]

    elif "http://" in top.uri:
        schema = "http://"
        top.uri = top.uri[7:]

    else:
        schema = "http://"

    # take credit ;)
    if 1:
        print("")
        print("           #         ##")
        print("         ##       ###")
        print("        ## ### ###")
        print("      ###########")
        print("     #############")
        print("    ################")
        print("    ##################")
        print("   #######  ############")
        print("  ######      ############")
        print("   ###          ############")
        print("\n----------------------------")
        print(" slasher.py     v1.1.0")
        print("  by k-ddev")
        print("")
        print("  A directory scanner\n")
    else:
        print("\n----------------------------")
    print("  Target host  - %s" % top.uri)
    print("  Wordlist     - %s" % list_path)
    print("  Requests     - %d" % concurrent)
    print("  Enabled modes:")
    if all_flag:
        print("    [*] scan all") 
    if list_flag:
        print("    > scan listable")
    print("----------------------------\n")

    # make sure top is even connectable
    print ("[*] Checking for remote host connection at [%s]" % (schema + top.uri))
    response = grequests.map([grequests.get(schema + top.uri)])[0]
    try:
        if response.status_code >= 200 and response.status_code < 300:
            print ("[*] Great Success!")

        else:
            print ("[!] No connection to [%s]\n[*] Exiting." % (schema + top.uri))
            sys.exit(1)
    except:
        print ("[!] No connection to [%s]\n[*] Exiting." % (schema + top.uri))
        sys.exit(1)

    # make sure the dictionary works
    print("\n[*] Checking dictionary: [%s]" % list_path)
    try:
        with open(list_path) as f:
            dictionary = f.read().strip().split('\n')
        print("[*] Successfully set up them the bomb with [%d] paths to try" % (len(dictionary)))
    except IOError:
        print("[!] Something went wrong :(")
        print("[*] Failed to read [%s]" % list_path)
        sys.exit(1)

    # main scan loop
    """
    Here's where our collection 'host' comes into play. Each index of host
    represents a level of the site. host[0] as seen already contains only
    the top page. host[1] will be each page below top, and host[2] will
    contain every subpage of every page in host[1], and so on. Our loop
    will scan every page in host[i], collecting found pages into host[i+1].
    If host[i+1] contains no pages, we know there's nothing else to find.   
    """
    print("\n[*] Starting scan.")
    depth = 0
    while True:
        # allocate a new index of host for any pages we find
        host.append([])
        try:
            assert len(host) == depth + 2
        except AssertionError:
            print("[!] Something weird happened. The greatest depth of the site as we know it")
            print("    should be 1 greater than the depth of the pages currently being scanned,")
            print("    but is not.")
            sys.exit(0)

        # scan every page on current depth and collect found pages
        for page in host[depth]:
            if all_flag or page.may_be_directory:
                if not list_flag and page.is_listable:
                    print("\r\033[K\n[*] skipped listable directory: %s" % page.get_url())
                    print("    specify -l to scan listable")

                else:
                    found = scan(page)
                    for f in found:
                        host[-1].append(f)

        # if no more pages were found, we're done. Otherwise, go to next depth
        if len(host[-1]) == 0:
            break

        else:
            depth += 1

    print("\r\033[K\n[*] All their subpage are belong to us!")
Example #59
0
def run():

    symbol2code = {}
    stock_map = {}
    for stock in main_session.query(models.DailyBasic).all():
        ts_code = stock.ts_code
        market = ts_code.split('.')[1].lower()
        symbol = ts_code.split('.')[0]
        code = '{market}{symbol}'.format(market=market, symbol=symbol)
        symbol2code[symbol] = code
        stock_map[code] = {'circ_mv': float(stock.circ_mv)}

    batch_size = 500
    req_list = []
    for i in range(0, len(target_symbols), batch_size):
        keys = []
        for symbol in target_symbols[i:i + batch_size]:
            query_key = symbol2code[symbol]
            keys.append(query_key)

        req_list.append(
            grequests.get('http://hq.sinajs.cn/list={}'.format(
                ','.join(keys))))

    while True:
        time_a = time.time()
        try:
            responses = grequests.map(req_list)
            print('====== {} ======'.format(
                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
            displays = []
            for response in responses:
                res = response.text.strip().split(';\n')
                for i in res:
                    j = i.split(',')
                    name = j[0].split('="')[1]
                    code = j[0].split('="')[0].split('_')[-1]
                    yesterday_closing_price = float(j[2])
                    current_price = float(j[3])
                    today_max_price = float(j[4])
                    buy_one_price = float(j[6])
                    buy_one_vol = float(j[10])
                    today_limit_price = round(yesterday_closing_price * 1.2, 2)
                    chg = (current_price / yesterday_closing_price - 1)
                    chg_display = '{}%'.format(round(chg * 100, 2))
                    circ_mv = stock_map[code]['circ_mv']

                    if_display = False
                    type = 1
                    if today_max_price == today_limit_price:  #摸过板的
                        if_display = True
                        if buy_one_price < today_limit_price:  #开板
                            if_display = True
                        elif buy_one_price * buy_one_vol < 10000000:  #封单小于1kw
                            if_display = True
                            type = 2

                    elif chg > 0.1:
                        if_display = True

                    # if_display = True

                    if if_display:
                        if type == 2:
                            displays.append({
                                'note':
                                '{code}\t{name}\tchg:{chg}\tprice:{price}\tcirc_mv:{circ_mv}亿\t封单:{vol}手'
                                .format(code=code,
                                        name=name,
                                        chg=chg_display,
                                        price=round(current_price, 2),
                                        circ_mv=int(circ_mv),
                                        vol=int(buy_one_vol / 100)),
                                'chg':
                                chg
                            })
                        else:
                            displays.append({
                                'note':
                                '{code}\t{name}\tchg:{chg}\tprice:{price}\tcirc_mv:{circ_mv}亿'
                                .format(code=code,
                                        name=name,
                                        chg=chg_display,
                                        price=round(current_price, 2),
                                        circ_mv=int(circ_mv)),
                                'chg':
                                chg
                            })

            displays.sort(key=lambda x: x['chg'], reverse=False)
            notes = [i['note'] for i in displays]
            print('\n'.join(notes))
        except Exception as e:
            print(e)
            continue
        time_b = time.time()
        cost = time_b - time_a
        time.sleep(1 - cost)
Example #60
0
def test_query_blockchain_events(
        api_backend,
        api_test_context,
        api_raiden_service):

    # Adding some mock events. Some of these events should not normally contain
    # a block number but for the purposes of making sure block numbers propagate
    # in the API logic I am adding them here and testing for them later.
    api_test_context.add_events([{
        '_event_type': 'TokenAdded',
        'block_number': 1,
        'channel_manager_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9',
        'token_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9'
    }, {
        '_event_type': 'TokenAdded',
        'block_number': 13,
        'channel_manager_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9',
        'token_address': '0xea674fdde714fd979de3edf0f56aa9716b898ec8'
    }, {
        '_event_type': 'ChannelNew',
        'settle_timeout': 10,
        'netting_channel': '0xea674fdde714fd979de3edf0f56aa9716b898ec8',
        'participant1': '0x4894a542053248e0c504e3def2048c08f73e1ca6',
        'participant2': '0x356857Cd22CBEFccDa4e96AF13b408623473237A',
        'block_number': 15,
    }, {
        '_event_type': 'ChannelNew',
        'settle_timeout': 10,
        'netting_channel': '0xa193fb0032c8635d590f8f31be9f70bd12451b1e',
        'participant1': '0xcd111aa492a9c77a367c36e6d6af8e6f212e0c8e',
        'participant2': '0x88bacc4ddc8f8a5987e1b990bb7f9e8430b24f1a',
        'block_number': 100,
    }, {
        '_event_type': 'ChannelNewBalance',
        'token_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9',
        'participant': '0xcd111aa492a9c77a367c36e6d6af8e6f212e0c8e',
        'balance': 200,
        'block_number': 20,
    }, {
        '_event_type': 'ChannelNewBalance',
        'token_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9',
        'participant': '0x00472c1e4275230354dbe5007a5976053f12610a',
        'balance': 650,
        'block_number': 150,
    }, {
        '_event_type': 'ChannelSettled',
        'block_number': 35,
    }, {
        '_event_type': 'ChannelSettled',
        'block_number': 250,
    }])

    # and now let's query the network events for 'TokenAdded' for blocks 1-10
    request = grequests.get(
        api_url_for(
            api_backend,
            'networkeventsresource',
            from_block=0,
            to_block=10
        )
    )
    response = request.send().response
    assert_proper_response(response)
    response = json.loads(response._content)
    assert len(response) == 1
    assert response[0] == {
        'event_type': 'TokenAdded',
        'block_number': 1,
        'channel_manager_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9',
        'token_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9'
    }

    # query ChannelNew event for a token
    api_test_context.specify_token_for_channelnew('0x61c808d82a3ac53231750dadc13c777b59310bd9')
    request = grequests.get(
        api_url_for(
            api_backend,
            'tokeneventsresource',
            token_address='0x61c808d82a3ac53231750dadc13c777b59310bd9',
            from_block=5,
            to_block=20
        )
    )
    response = request.send().response
    assert_proper_response(response)
    response = json.loads(response._content)
    assert len(response) == 1
    assert response[0] == {
        'event_type': 'ChannelNew',
        'settle_timeout': 10,
        'netting_channel': '0xea674fdde714fd979de3edf0f56aa9716b898ec8',
        'participant1': '0x4894a542053248e0c504e3def2048c08f73e1ca6',
        'participant2': '0x356857Cd22CBEFccDa4e96AF13b408623473237A',
        'block_number': 15,
    }

    # finally query for some channel related events
    # Note: No need to test em all since this does not test the implementation
    # of `get_channel_events()` but just makes sure the proper data make it there
    api_test_context.specify_channel_for_events('0xedbaf3c5100302dcdda53269322f3730b1f0416d')
    request = grequests.get(
        api_url_for(
            api_backend,
            'channeleventsresource',
            channel_address='0xedbaf3c5100302dcdda53269322f3730b1f0416d',
            from_block=10,
            to_block=90
        )
    )
    response = request.send().response
    assert_proper_response(response)
    response = json.loads(response._content)
    assert len(response) == 2
    assert response[0] == {
        'event_type': 'ChannelNewBalance',
        'token_address': '0x61c808d82a3ac53231750dadc13c777b59310bd9',
        'participant': '0xcd111aa492a9c77a367c36e6d6af8e6f212e0c8e',
        'balance': 200,
        'block_number': 20,
    }
    assert response[1] == {
        'event_type': 'ChannelSettled',
        'block_number': 35,
    }