示例#1
0
def TwitterScan(encoded_url):

    # This tries to request the api, if successful extract the json
    try:
        twitter_req = urllib.urlopen("http://urls.api.twitter.com/1/urls/count.json?url={url}".format(url=encoded_url))

    # Exception occured with urllib.urlopen(), store it in the Error model
    except Exception as e:

        error = Error(
            error="URL: {url}\nTwitter API error: {error}".format(error=e, url=encoded_url), created_by="social_sync.py"
        )
        error.save()

        # Print that it failed
        print " This one failed! Automatically saved in errors model"

        # Return False so the Command function can know it failed.
        return False

    # Successfully requested the API, transform to JSON and return the total interactions
    twitter_json = json.load(twitter_req)
    return twitter_json["count"]
示例#2
0
def contact_view(request):
    if request.method == "POST":
        contact_form = ContactForm(request.POST or None)
        if contact_form.is_valid():
            data = contact_form.cleaned_data
            city = data.get("city")
            language = data.get("language")
            email = data.get("email")
            qs = Error.objects.filter(timestamp=dt.date.today())
            if qs.exists():
                err = qs.first()
                data = err.data.get("user_data", [])
                data.append({"city": city, "language": language, "email": email})
                err.data["user_data"] = data
                err.save()
            else:
                data = [{"city": city, "language": language, "email": email}]
                Error(data=f"user_data:{data}").save()
            messages.success(request, 'Данные отправлены')
            return redirect("accounts:update")
        else:
            return redirect("accounts:update")
    else:
        return redirect("accounts:login")
示例#3
0
def FacebookScan(encoded_url, use_proxies):

    # This checks if the user said that it wants to sync using proxies
    if use_proxies:
        # This assings the maximum time in seconds to wait for the urllib.urlopen() below to
        # respond
        socket.setdefaulttimeout(15)

        # This gets the proxy that was used longer ago from the FreeProxy model.
        # It also returns the proxy IP in a dictionary to use with urllib.urlopen
        oldest_proxy, oldest_proxy_dic = OldestProxyDic()

        # The function to get the proxies failed, without proxies this function can't continue
        # return false.
        if oldest_proxy is False:
            return False

    # If this runs it means the user didn't select any proxy.
    # Just assign an empty dictionary, that way urllib.openurl() will ignore the proxies
    # argument and use localhost
    else:
        oldest_proxy_dic = {"": ""}

    # We have to use this because if the URL contains things like "?X=" the facebook API
    # will think we are passing parameters to it.
    encoded_url = urllib.quote(encoded_url)

    # This tries to request the Facebook API. If successful, it will extract the JSON.
    # If it fails it will return False to let the Command function know it did so.
    try:

        # IMPORTANT NOTE: The idea to use proxies was to bypass the API request limit.
        # If we are using proxies + an access_token, it makes no sense to use proxies but
        # for now we have to use an access_token until a new function is coded for the proxies.
        # The reason for that is, the facebook API json changes when you visit it without an
        # access token, that means the json is different and so a new function is required.
        facebook_req = urllib.urlopen(
            "https://graph.facebook.com/v2.3/{url}?access_token="
            "CONFIGURE-ME: Add your own Facebook access token".format(url=encoded_url),
            proxies=oldest_proxy_dic,
        )

    # The urllib.urlopen() returned an exception, store it in the Error model
    except Exception as e:

        error = Error(
            error="URL: {url}\nurllib.urlopen() error: {error}".format(error=e, url=encoded_url),
            created_by="social_sync.py",
        )
        error.save()

        # This checks if the exception was caused for any of the following reasons
        # If it did, call the DisableProxy function to mark the used proxy as obsolete.
        if "[Errno socket error]" in str(e) or "http protocol error" in str(e):
            DisableProxy(oldest_proxy, e)

        # Tell the user it failed
        print " This one failed! Automatically saved in errors model"

        # Return false instead of the social interactions
        return False

    # This checks if the returned html is actually a json. If it's not json, the proxy works,
    # if it isn't, the proxy is obsolete
    try:
        facebook_json = json.load(facebook_req)

    except Exception as e:

        error = Error(
            error=("URL: {url}\n" "json.load() error: {error}\n" "Proxy used during error: {proxy_ip}").format(
                error=e, url=encoded_url, proxy_ip=oldest_proxy.ip
            ),
            created_by="social_sync.py",
        )
        error.save()

        # This proxy is obsolete, call DisableProxy to mark it as such in the FreeProxy model
        DisableProxy(oldest_proxy, e)

        # Tell the user it failed
        print " This one failed! Automatically saved in errors model"

        # Tell the Command function it failed
        return False

    # This checks if there's a key called error in the facebook json.
    # If so, the facebook api returned an error.
    if "error" in facebook_json:

        # Tell the user the API returned an exception and the exception message
        print " Facebook API returned exception:"
        print "  {exception}".format(exception=facebook_json["error"]["message"])

        # Store the error in the Error model
        error = Error(
            error="URL: {url}\nFacebook API returned exception:\n{error}".format(
                error=facebook_json["error"]["message"], url=encoded_url
            ),
            created_by="social_sync.py",
        )
        error.save()

        # Return False instead of the total interactions
        return False

    # This checks if the Facebook API returned nothing for the requested URL
    # This can happen if the URL is from facebook itself. I don't know why the facebook API
    # does this.
    if not "share" in facebook_json:

        # Store the error in the errors model, it will probably spam the error model with this
        # for a couple of days but it doesn't hurt.
        error = Error(
            error="URL: {url}\n"
            "The Facebook API returned no shares.\n"
            "This is known to happen if the URL in question is from Facebook itself".format(url=encoded_url),
            created_by="social_sync.py",
        )
        error.save()

        # Tell the user what happend, never bad practice
        print " The Facebook API returned no shares, setting it to 0"

        # It returns 0 because this way it will not delete the social interaction entry.
        # We can't get the facebook interactions for this URL but it's still of
        # interest to get the twitter ones.
        return 0

    # PLEASE READ BEFORE ADDING MORE CODE BELOW:
    # it is important that "if not 'share' in facebook_json" stays at the bottom.
    # Because this 'if' returns 0 if true, it is important that the other 'ifs' to detect
    # errors run first.

    # Everything went fine; return the total interactions
    return facebook_json["share"]["share_count"]
示例#4
0
for vacancy in vacs:
    v = Vacancy(**vacancy)
    print(vacancy)
    try:
        v.save()
    except DatabaseError:
        pass

if errors:
    qs = Error.objects.filter(timestamp=dt.date.today())
    if qs.exists:
        err = qs.first()
        err.data.update({'errors': errors})
        err.save()
    else:
        er = Error(data=f'errors: {errors}').save()
print(time.time() - start)

# city = City.objects.filter(slug="moscow").first()
# language = Language.objects.filter(slug="python").first()
# vacs_hh, er_hh = hh()
# vacs_jooble, er_job = jooble()
# vacs_indeed, er_ind = indeed()
# vacs = vacs_hh + vacs_jooble + vacs_indeed
# errs = er_hh + er_job + er_ind

# print(vacs)
# print(errs)
# print(er_hh)
# with open("../vacs_hh.txt", "w", encoding="utf8") as f:
#    f.write("{}".format(vacs_hh))