Exemplo n.º 1
0
def put_resource(url, data):
    """
    :param url:
    :param data:
    :return: Python data, or False if the request failed.
    """
    resource = None
    headers = {'Content-Type': 'application/json'}
    response = requests.put(url, json=data, auth=get_auth(), headers=headers)
    if response.status_code == 429:
        print('Rate limited! Please wait.')
        time.sleep(int(response.headers['retry-after']))
        response = requests.post(url,
                                 json=data,
                                 auth=get_auth(),
                                 headers=headers)
    if response.status_code != 200:
        print('Failed to update record with error {}:'.format(
            response.status_code))
        print(response.text)
        return False
    for k, v in response.json().items():
        resource = v
    if type(resource) is dict:
        return resource
    return None
Exemplo n.º 2
0
def get_resource_list(url, list_name=None, paginate=True):
    """
    Returns a list of HC resources specified by the url basename (such as .../articles.json)
    :param url: A full endpoint url, such as 'https://support.zendesk.com/api/v2/help_center/articles.json'
    :param list_name: The list name in the response per the docs. Required if list name not the same as resource name
    :param paginate: Whether the endpoint has pagination (i.e., a 'next_page' property). Example false: missing translations
    :return: List of resources, or False if the request failed
    """
    if list_name:
        resource = list_name
    else:
        resource = Path(url).stem
    record_list = {resource: []}
    while url:
        subdomain = urlparse(url).hostname.split('.')[0]
        response = requests.get(url, auth=get_auth(subdomain))
        if response.status_code == 429:
            print('Rate limited! Please wait.')
            time.sleep(int(response.headers['retry-after']))
            response = requests.get(url, auth=get_auth())
        if response.status_code != 200:
            print('Error with status code {}'.format(response.status_code))
            print(response.text)
            return False
        data = response.json()
        if data[resource]:  # guard against empty record list
            record_list[resource].extend(data[resource])
        if paginate:
            url = data['next_page']
        else:
            break
    return record_list[resource]
Exemplo n.º 3
0
	def authorize(self, subaddr):
		gsm.at_connect(self.PROXY_IP, self.USERNAME, self.PASSWORD)
		auth = get_auth(gsm.CNUM, "", b'26485001') # Retrieve a server nonce
		self.at_start_http_session()
		server_nonce = yield from self.http_action("GET", self.url(subaddr), time=self.timestamp(), auth=auth) # Retrieve servernonce
		auth = get_auth(gsm.CNUM, server_nonce, b'42849900')

		return auth
Exemplo n.º 4
0
def get_resource_list(url):
    """
    Returns a list of HC resources specified by the url basename (such as .../articles.json)
    :param url: A full endpoint url, such as 'https://support.zendesk.com/api/v2/help_center/articles.json'
    :return: List of resources, or False if the request failed.
    """
    session = requests.Session()
    session.auth = get_auth()

    o = urlparse(url)
    resource = os.path.splitext(os.path.basename(
        o.path))[0]  # e.g., 'articles'
    record_list = {resource: []}
    while url:
        response = session.get(url)
        if response.status_code == 429:
            print('Rate limited! Please wait.')
            time.sleep(int(response.headers['retry-after']))
            response = session.get(url)
        if response.status_code != 200:
            print('Error with status code {}'.format(response.status_code))
            exit()
        data = response.json()
        if data[resource]:  # guard against empty record list
            record_list[resource].extend(data[resource])
        url = data['next_page']
    return record_list[resource]
Exemplo n.º 5
0
def delete_resource(url):
    """
    Runs a DELETE request on any Delete endpoint in the Zendesk API
    :param url: A full endpoint url, such as 'https://support.zendesk.com/api/v2/help_center/articles/2342572.json'
    :return: If successful, a 204 status code. If not, None
    """
    response = requests.delete(url, auth=get_auth())
    if response.status_code == 429:
        print('Rate limited! Please wait.')
        time.sleep(int(response.headers['retry-after']))
        response = requests.delete(url, auth=get_auth())
    if response.status_code != 204:
        print('Failed to delete record with error {}'.format(response.status_code))
        print(response.text)
        return False
    return None
Exemplo n.º 6
0
def generate_headers(request, method, url):
    code = 200
    content_type = "text/html"
    if method not in ('GET', 'POST'):
        code = 405
    elif re.match(r'^/static', url):
        if os.path.isfile(url[1:]):
            subp = url[8:]
            content_type = get_content_type(subp)

        else:
            code = 404
    else:
        if re.match(r'^/json', url):
            content_type = 'application/json'
        if method == 'GET' and url not in URLS_GET and url not in URLS_GET_ajax:
            code = 404
        elif method == 'POST' and url not in URLS_POST:
            code = 404
        code2 = get_auth(request)
        code = code2 if code == 200 else code

    request.response.code = code
    request.response.content_type = content_type
    return code
Exemplo n.º 7
0
    def __init__(self, db_config,
                 measure_timestamp_diff=False,
                 blocking_auth=True,
                 compression=None,
                 verbose=10):

        guest = db_config.get('guest')

        self.app = pyrebase.initialize_app(db_config)

        if compression is None:
            compression = db_config.get('compression')

        self.auth = None
        if not guest and 'serviceAccount' not in db_config.keys():
            self.auth = get_auth(self.app,
                                 db_config.get("use_email_auth"),
                                 db_config.get("email"),
                                 db_config.get("password"),
                                 blocking_auth)

        self.logger = logs.getLogger('FirebaseArtifactStore')
        self.logger.setLevel(verbose)
        super(FirebaseArtifactStore, self).__init__(
            measure_timestamp_diff,
            compression=compression)
Exemplo n.º 8
0
def get_resource(url):
    """
    Returns a single HC resource
    :param url: A full endpoint url, such as 'https://support.zendesk.com/api/v2/help_center/articles/2342572.json'
    :return: Dict of a resource, or False if the request failed.
    """
    resource = None
    response = requests.get(url, auth=get_auth())
    if response.status_code == 429:
        print('Rate limited! Please wait.')
        time.sleep(int(response.headers['retry-after']))
        response = requests.get(url, auth=get_auth())
    if response.status_code != 200:
        print('Failed to get record with error {}:'.format(response.status_code))
        print(response.text)
        return False
    for k, v in response.json().items():
        resource = v
    if type(resource) is dict:
        return resource
    return None
Exemplo n.º 9
0
def retrieve_data_gen(username,
                      password,
                      template,
                      query_args=None,
                      single_request=False):
    auth = get_auth(username, password)
    query_args = get_query_args(query_args)

    logger.info(f"The auth for the user is {auth}")
    per_page = 100
    page = 0

    while True:
        page = page + 1
        request = construct_request(per_page, page, template, auth)  # noqa
        r, errors = get_response(request, auth, template)

        status_code = int(r.getcode())

        retries = 0
        while retries < 3 and status_code == 502:
            print(
                'API request returned HTTP 502: Bad Gateway. Retrying in 5 seconds'
            )
            retries += 1
            time.sleep(5)
            request = construct_request(per_page, page, query_args, template,
                                        auth)  # noqa
            r, errors = get_response(request, auth, template)

            status_code = int(r.getcode())

        if status_code != 200:
            template = 'API request returned HTTP {0}: {1}'
            errors.append(template.format(status_code, r.reason))
            logger.error(errors)

        response = json.loads(r.read().decode('utf-8'))
        if len(errors) == 0:
            if type(response) == list:
                for resp in response:
                    yield resp
                if len(response) < per_page:
                    break
            elif type(response) == dict and single_request:
                yield response

        if len(errors) > 0:
            logger.error(errors)

        if single_request:
            break
Exemplo n.º 10
0
def configure():
	get_event_loop().run_until_complete(gsm.at_connect(PROXY_IP, USERNAME, PASSWORD))
	auth = get_auth(gsm.CNUM, "", b'26485001')
	get_event_loop().run_until_complete(gsm.start_http_session(PROXY_IP))
	server_nonce = yield from gsm.http_action("GET", url(server_url, port, "conf"), time=timestamp(), auth=auth)
	get_event_loop().run_until_complete(gsm.terminate_http_session())
	auth = get_auth(gsm.CNUM, server_nonce, b'42849900')

	get_event_loop().run_until_complete(gsm.start_http_session(PROXY_IP))
	conf = yield from gsm.http_action("GET", url(server_url, port, "conf"), time=timestamp(), auth=auth)

	get_event_loop().run_until_complete(gsm.terminate_http_session())
	get_event_loop().run_until_complete(gsm.at_disconnect())

	# Parse configuration settings
	if conf:
		track_location = parse_conf("track_location", conf)
		track_state = parse_conf("track_state", conf)
		measure_ms = parse_conf("measure_interval", conf)
		stream_ms = parse_conf("stream_interval", conf)
		fixed_state_track = parse_conf("fixed_state_track", conf)
		verbose = parse_conf("verbose", conf)
Exemplo n.º 11
0
def stream(stream_ms):
	while True:
		yield from sleep(stream_ms)
		print("Authorizing...")
		auth = get_auth(gsm.CNUM, "", b'26485001')
		get_event_loop().call_soon(gsm.at_connect(PROXY_IP, USERNAME, PASSWORD))
		print("Preauth: %s" %auth)
		print("Initializing HTTP session...")
		get_event_loop().call_soon(gsm.start_http_session(PROXY_IP))
		print("Retrieving server nonce...")
		get_event_loop().call_soon(gsm.start_http_session(PROXY_IP))
		server_nonce = yield from gsm.http_action("GET", url(server_url, port, "stream"), time=timestamp(), auth=auth)
		print("Possible server nonce: %s" %server_nonce[:])
		if server_nonce and not "CME" in server_nonce:
			print("Succesfully obtained server nonce %s" %server_nonce)
			auth = get_auth(gsm.CNUM, server_nonce, b'42849900')
		else:
			print("Unable to obtain server nonce")

		print("Stopping.")
		get_event_loop().call_soon(gsm.terminate_http_session())

		print("Running production authorization.")
		coords = "test"
		state = "test"
		data = '{"auth":%s, %s, ' %(auth, coords)
		if track_state:
			data += '"state":%s, ' %state
		data += '"time":%s}' %(timestamp())
		print(data)
		print("Posting location data...")

		get_event_loop().call_soon(gsm.start_http_session(PROXY_IP))
		post = yield from gsm.http_action("POST", url(server_url, port, "stream"), data=data)
		get_event_loop().call_soon(gsm.terminate_http_session())
		get_event_loop().call_soon(gsm.at_disconnect())
Exemplo n.º 12
0
	def configure(self):
		""" Daily configuration call. gsm.at_id() must be performed, as well as at_connect """

		gsm.at_connect(self.PROXY_IP, self.USERNAME, self.PASSWORD)
		auth = get_auth(gsm.CNUM, "", b'26485001') # Retrieve a server nonce
		# urc add event "CONNECT OK"
		self.at_start_http_session()
		server_nonce = yield from self.http_action("GET", self.url("conf"), time=self.timestamp(), auth=auth) # Retrieve servernonce

		auth = get_auth(gsm.CNUM, server_nonce, b'42849900')
		# urc event del and if everything worked out fine,
		# Obtain configuration
		conf = yield from self.http_action("GET", self.url("conf"), time=self.timestamp(), auth=auth)

		self.at_terminate_http_session() # TODO: folded with individual requests or like this?

		self.at_disconnect()

		self.track_location = parse_conf("track_location", conf)
		self.track_state = parse_conf("track_state", conf)
		self.measure_ms = parse_conf("measure_interval", conf)
		self.stream_ms = parse_conf("stream_interval", conf)
		self.fixed_state_track = = parse_conf("fixed_state_track", conf)
		self.verbose = parse_conf("verbose", conf)
Exemplo n.º 13
0
def post_resource(url, data, status=201):
    """
    :param url:
    :param data:
    :param status: HTTP status. Normally 201 but some POST requests return 200
    :return: Python data, or False if the request failed.
    """
    resource = None
    headers = {'Content-Type': 'application/json'}
    subdomain = urlparse(url).hostname.split('.')[0]
    response = requests.post(url, json=data, auth=get_auth(subdomain), headers=headers)
    if response.status_code == 429:
        print('Rate limited! Please wait.')
        time.sleep(int(response.headers['retry-after']))
        response = requests.post(url, json=data, auth=get_auth(), headers=headers)
    if response.status_code != status:
        print('Failed to create record with error {}:'.format(response.status_code))
        print(response.text)
        return False
    for k, v in response.json().items():
        resource = v
    if type(resource) is dict:
        return resource
    return None
Exemplo n.º 14
0
def get_github_repo_url(username, password, repository, prefer_ssh=True):
    # if args.prefer_ssh:
    #     return repository['ssh_url']
    if repository.get('is_gist'):
        return repository['git_pull_url']

    if prefer_ssh:
        return repository['ssh_url']

    ##if its a private url
    auth = get_auth(username, password, False)
    if auth:
        logger.info(f"Auth is prsent {auth}")
        repo_url = 'https://{0}@{1}/{2}/{3}.git'.format(
            auth, get_github_host(), repository['owner']['login'],
            repository['name'])
    else:
        repo_url = repository['clone_url']

    return repo_url
Exemplo n.º 15
0
    def __init__(self,
                 config,
                 verbose=10,
                 blocking_auth=True,
                 compression=None):
        # TODO: implement connection
        self.url = config.get('serverUrl')
        self.verbose = verbose
        self.logger = logs.getLogger('HTTPProvider')
        self.logger.setLevel(self.verbose)

        self.auth = None
        self.app = pyrebase.initialize_app(config)
        guest = config.get('guest')
        if not guest and 'serviceAccount' not in config.keys():
            self.auth = get_auth(self.app, config.get("use_email_auth"),
                                 config.get("email"), config.get("password"),
                                 blocking_auth)

        self.compression = compression
        if self.compression is None:
            self.compression = config.get('compression')
Exemplo n.º 16
0
def get_results_page():
    q = request.args.get('search')
    tweets = search(q, 10)
    
#Mongo Database
    class MyStreamListener(StreamListener):
        # Class that Tweepy has built, has collects all functionality of tweets
    
        def __init__(self):
            super(MyStreamListener, self).__init__()
            self.num_tweets = 0
    
        def on_data(self, data):
            if self.num_tweets < 10:
                self.num_tweets += 1
                try:
                    with MongoClient(MONGODB_URI) as conn:
                        db = conn[MONGODB_NAME]
                        coll = db[q]
                        coll.insert(json.loads(data))
                        return True
                except BaseException as e:
                    print ("Failed on_data: %s" % str(e))
                    return True
            else:
                return False
    
        def on_error(self, status):
            print(status)
            return True
    
    auth = get_auth()
    
    twitter_stream = Stream(auth, MyStreamListener())
    twitter_stream.filter(track=q)


    return render_template("results.html", tweets = tweets, q = q)
Exemplo n.º 17
0
    def __init__(
            self,
            db_config,
            blocking_auth=True,
            verbose=10,
            store=None,
            compression=None):
        guest = db_config.get('guest')

        self.app = pyrebase.initialize_app(db_config)
        self.logger = logs.getLogger(self.__class__.__name__)
        self.logger.setLevel(verbose)

        self.compression = compression
        if self.compression is None:
            self.compression = db_config.get('compression')

        self.auth = None
        if not guest and 'serviceAccount' not in db_config.keys():
            self.auth = get_auth(self.app,
                                 db_config.get("use_email_auth"),
                                 db_config.get("email"),
                                 db_config.get("password"),
                                 blocking_auth)

        self.store = store if store else FirebaseArtifactStore(
            db_config,
            verbose=verbose,
            blocking_auth=blocking_auth,
            compression=self.compression
        )

        if self.auth and not self.auth.expired:
            self.register_user(None, self.auth.get_user_email())

        self.max_keys = db_config.get('max_keys', 100)
Exemplo n.º 18
0
def lyrics(bot, update, user_data):
    user_id = str(update.message.chat_id) + '_' + update.message.from_user['username']
    logging.info("Starting lyrics for user_id: {}".format(user_id))
    sp_oauth = auth.get_auth(username=user_id, scope=scope)
    token_info = sp_oauth.get_cached_token()
    # If there is no cached token, ask for it
    if not token_info:
        reply_text = "Token unavaliable. Please visit {} and paste/share the url you obtain (https://localhost...).".format(sp_oauth.get_authorize_url())
        update.message.reply_text(reply_text)
        user_data['sp_oauth'] = sp_oauth
        logging.info("text: %s, reply: %s", update.message.text, reply_text)
        return TYPING_URL
    else:
        token = token_info['access_token']
        current = get_currently_playing(token)
        if not current:
            reply_text = "No song currently playing."
            update.message.reply_text(reply_text)
            logging.info("text: %s, reply: %s", update.message.text, reply_text)
            return ConversationHandler.END
        item = current['item']
        if not item:
            reply_text = "No song detected. Perhaps it's a podcast?"
            update.message.reply_text(reply_text)
            logging.info("text: %s, reply: %s", update.message.text, reply_text)
            return ConversationHandler.END
        name = item['name']
        artist = item['artists'][0]['name']
        l = get_lyrics_genius(song=name, artist=artist)
        if l:
            reply_text = "Song: {}\nArtist: {}\n{}".format(name, artist, l)
        else:
            reply_text = "Sorry, no lyrics found for song: {}, artist: {}".format(name, artist)
        update.message.reply_text(reply_text)
        logging.info("text: %s, reply: %s", update.message.text, reply_text)
        return ConversationHandler.END
Exemplo n.º 19
0
 def __init__(self):
     self.auth = auth.get_auth()
     self.commands = commands
     self.client = BotClient(partial(self.handle_message))
Exemplo n.º 20
0
#!/usr/bin/env python
# -*- coding: utf-8 -*-

from auth import get_auth
import json

q = "Atlas Shrugged"

count = 100

twitter_api = get_auth()

search_results = twitter_api.search.tweets(q=q, count=count)

statuses = search_results['statuses']

for _ in range(5):
    print("Length of statuses", len(statuses))
    next_results = ""
    try:
        next_results = search_results['search_metadata']['next_results']
        print("got next results")
    except KeyError as e:
        print("no more results")
        break

    kwargs = dict([kv.split('=') for kv in next_results[1:].split("&")])

    #print( "next_results: ", next_results )
    #print( "kwargs: ", kwargs )
Exemplo n.º 21
0
def post_attachment(url, attachment, status=201):
    resource = None
    file = get_resource(attachment['content_url'], attachment=True)
    subdomain = urlparse(url).hostname.split('.')[0]
    file_path = os.path.join('attachments', attachment['file_name'])
    with open(file_path, 'wb') as f:
        f.write(file)
    with open(file_path, 'rb') as u:
        response = requests.post(url, data={'inline': attachment['inline']}, files={'file':u}, auth=get_auth(subdomain))
        if response.status_code == 429:
            print('Rate limited! Please wait.')
            time.sleep(int(response.headers['retry-after']))
            response = requests.post(url, files={'inline':attachment['inline'], 'file':u})
        if response.status_code != status:
            print('Failed to create record with error {}:'.format(response.status_code))
            print(response.text)
            return False
        for k, v in response.json().items():
            resource = v
        if type(resource) is dict:
            return resource
        return None
Exemplo n.º 22
0
from aip import AipFace
import base64, auth, json, urllib, urllib2

APP_ID = '10440639'
API_KEY = 'REeeig5V9NpTkffDiqvTSyuV'
SECRET_KEY = 'oUdGrgbktr8G5BpirKwnqMdscGlGV3wu'

client = AipFace(APP_ID, API_KEY, SECRET_KEY)

content = auth.get_auth()
content = eval(content)
print content['access_token']
request_url = 'https://aip.baidubce.com/rest/2.0/face/v1/detect'
f = open('face.jpg', 'rb')
img = base64.b64encode(f.read())
print img
params = {
    "face_fields":
    "age,beauty,expression,faceshape,gender,glasses,landmark,race,qualities",
    "image": img,
    "max_face_num": 5
}
params = urllib.urlencode(params)
request_url = request_url + "?access_token=" + content['access_token']
request = urllib2.Request(url=request_url, data=params)
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
response = urllib2.urlopen(request)
result = eval(response.read())
print('age:%s' % result['result'][0]['age'])
print('beauty:%s' % result['result'][0]['beauty'])
print('gender:%s' % result['result'][0]['gender'])
Exemplo n.º 23
0
import requests
from auth import get_auth

MY_URL_NUM = '2361076'
AUTH = get_auth()
BASE_URL = 'https://basecamp.com/{}/api/v1/'
MY_URL = BASE_URL.format(MY_URL_NUM)


class BCError(Exception):
    pass


class BaseCampConnectionError(BCError):
    pass


class BaseCampConnectionAuthError(BCError):
    pass


class BaseCamp(object):
    BASE_URL = 'https://basecamp.com/{}/api/v1/'

    def __init__(self, url_num, user=None, pw=None):
        self.base_url = BaseCamp.BASE_URL.format(url_num)
        if user is not None and pw is not None:
            self.auth = (user, pw)
        else:
            self.auth = ''
Exemplo n.º 24
0
def main():
    while True:
        handler()
        time.sleep(get_auth('monitor_interval'))
def stream_tweets(query, count):
    auth = get_auth()
    tweets = []
    twitter_stream = Stream(auth, MyStreamListener(count, tweets))
    twitter_stream.filter(track=[query])
    return tweets
Exemplo n.º 26
0
import requests
from auth import get_auth

MY_URL_NUM = '2361076'
AUTH = get_auth()
BASE_URL = 'https://basecamp.com/{}/api/v1/'
MY_URL = BASE_URL.format(MY_URL_NUM)

class BCError(Exception):
    pass

class BaseCampConnectionError(BCError):
    pass

class BaseCampConnectionAuthError(BCError):
    pass

class BaseCamp(object):
    BASE_URL = 'https://basecamp.com/{}/api/v1/'
    def __init__(self,url_num,user=None,pw=None):
        self.base_url = BaseCamp.BASE_URL.format(url_num)
        if user is not None and pw is not None:
            self.auth = (user,pw)
        else:
            self.auth = ''

    def connect(self):
        if not self.auth:
            raise BaseCampConnectionAuthError
        
Exemplo n.º 27
0
#!/usr/bin/env python3

import json
import os
import sys
import time

import requests

import auth

article_limit = None
if len(sys.argv) > 1:
    article_limit = sys.argv[1]

rootDir = 'html'
for dirName, subdirList, fileList in os.walk(rootDir):
    for fname in fileList:
        article, extension = os.path.splitext(fname)
        if extension == '.html':
            if article_limit is None or article_limit == article:
                url = 'https://memsource.zendesk.com/api/v2/help_center/articles/{}/translations/en-us.json'.format(
                    article)
                print(f"Requesting {url}")
                headers = {'Content-Type': 'application/json'}
                f = open(f"{dirName}/{fname}", "r")
                data = {"translation": {"body": f.read()}}
                response = requests.put(url, auth=auth.get_auth(), headers=headers, data=json.dumps(data))
                print(response)
                time.sleep(0.1)
Exemplo n.º 28
0
import os

import requests
import auth

# ----- Variables ----- #

auth = auth.get_auth()
loader_path = os.path.join('maps', 'loader.json')
unloader_path = os.path.join('maps', 'unloader.json')
global_map_path = os.path.join('maps', 'global_map.json')

# ----- Functions ----- #


def get_translation(article_id, hc, locale='en-us'):
    """
    :param article_id: Help Center article id
    :param hc: Help Center subdomain. Example: mondocam in mondocam.zendesk.com/hc
    :param locale: The article locale
    :return: A JSON object with the translation properties
    """
    url = 'https://{}.zendesk.com/api/v2/help_center/articles/{}/translations/{}.json'.format(
        hc, article_id, locale)
    response = requests.get(url, auth=auth)
    if response.status_code == 200:
        return response.json()['translation']
    else:
        print('Failed to get article {} with error {}'.format(
            article_id, response.status_code))
        return None
class MyStreamListener(StreamListener):

    def __init__(self):
        super(MyStreamListener, self).__init__()
        self.num_tweets = 0

    def on_data(self, data):
        if self.num_tweets < limit:
            self.num_tweets += 1
            try:
                with open('new_tweet_mining.json', 'a') as tweet_file:
                    tweet_file.write(data)
                    return True
            except BaseException as e:
                print ("Failed on_data: %s" % str(e))
            return True
        else:
            return False

    def on_error(self, status):
        print(status)
        return True

auth = get_auth()

twitter_stream = Stream(auth, MyStreamListener())
twitter_stream.filter(track=keyword_list)


Exemplo n.º 30
0
limit = 1000
filename = "repealNo.json"


class MyStreamListener(StreamListener):
    def __init__(self):
        super(MyStreamListener, self).__init__()
        self.num_tweets = 0

    def on_data(self, data):
        if self.num_tweets < limit:
            self.num_tweets += 1
            try:
                with open(filename, 'a') as tweet_file:
                    tweet_file.write(data)
                    return True
            except BaseException as e:
                print("Failed on_data: %s" % str(e))
            return True
        else:
            return False

    def on_error(self, status):
        print(status)
        return True


auth = get_auth()

twitter_stream = Stream(auth, MyStreamListener())
twitter_stream.filter(track=keyword_list)
Exemplo n.º 31
0
import json
import time
import pathlib

import auth

json_props = ['html_url', 'title']
page = 1

while True:

    url = 'https://memsource.zendesk.com/api/v2/help_center/en-us/articles.json?page={}'.format(
        page)
    print(f"Requesting {url}")
    headers = {'Content-Type': 'application/json'}
    response = requests.get(url, auth=auth.get_auth(), headers=headers)

    data = json.loads(response.text)

    if len(data['articles']) == 0:
        break

    for article in data["articles"]:
        dirname = 'html/{}/{}'.format(article['section_id'], article['id'])
        pathlib.Path(dirname).mkdir(parents=True, exist_ok=True)
        html_filename = '{}/{}.html'.format(dirname, article['id'])
        json_filename = '{}/{}.json'.format(dirname, article['id'])
        print(f"Writing {html_filename}")
        f = open(html_filename, 'w')
        body = article['body'] if article['body'] else ""
        f.write(body)