def put_resource(url, data):
"""
:param url:
:param data:
:return: Python data, or False if the request failed.
"""
resource = None
headers = {'Content-Type': 'application/json'}
response = requests.put(url, json=data, auth=get_auth(), headers=headers)
if response.status_code == 429:
print('Rate limited! Please wait.')
time.sleep(int(response.headers['retry-after']))
response = requests.post(url,
json=data,
auth=get_auth(),
headers=headers)
if response.status_code != 200:
print('Failed to update record with error {}:'.format(
response.status_code))
print(response.text)
return False
for k, v in response.json().items():
resource = v
if type(resource) is dict:
return resource
return None
def get_resource_list(url, list_name=None, paginate=True):
"""
Returns a list of HC resources specified by the url basename (such as .../articles.json)
:param url: A full endpoint url, such as 'https://support.zendesk.com/api/v2/help_center/articles.json'
:param list_name: The list name in the response per the docs. Required if list name not the same as resource name
:param paginate: Whether the endpoint has pagination (i.e., a 'next_page' property). Example false: missing translations
:return: List of resources, or False if the request failed
"""
if list_name:
resource = list_name
else:
resource = Path(url).stem
record_list = {resource: []}
while url:
subdomain = urlparse(url).hostname.split('.')[0]
response = requests.get(url, auth=get_auth(subdomain))
if response.status_code == 429:
print('Rate limited! Please wait.')
time.sleep(int(response.headers['retry-after']))
response = requests.get(url, auth=get_auth())
if response.status_code != 200:
print('Error with status code {}'.format(response.status_code))
print(response.text)
return False
data = response.json()
if data[resource]: # guard against empty record list
record_list[resource].extend(data[resource])
if paginate:
url = data['next_page']
else:
break
return record_list[resource]
def authorize(self, subaddr):
gsm.at_connect(self.PROXY_IP, self.USERNAME, self.PASSWORD)
auth = get_auth(gsm.CNUM, "", b'26485001') # Retrieve a server nonce
self.at_start_http_session()
server_nonce = yield from self.http_action("GET", self.url(subaddr), time=self.timestamp(), auth=auth) # Retrieve servernonce
auth = get_auth(gsm.CNUM, server_nonce, b'42849900')
return auth
def get_resource_list(url):
"""
Returns a list of HC resources specified by the url basename (such as .../articles.json)
:param url: A full endpoint url, such as 'https://support.zendesk.com/api/v2/help_center/articles.json'
:return: List of resources, or False if the request failed.
"""
session = requests.Session()
session.auth = get_auth()
o = urlparse(url)
resource = os.path.splitext(os.path.basename(
o.path))[0] # e.g., 'articles'
record_list = {resource: []}
while url:
response = session.get(url)
if response.status_code == 429:
print('Rate limited! Please wait.')
time.sleep(int(response.headers['retry-after']))
response = session.get(url)
if response.status_code != 200:
print('Error with status code {}'.format(response.status_code))
exit()
data = response.json()
if data[resource]: # guard against empty record list
record_list[resource].extend(data[resource])
url = data['next_page']
return record_list[resource]
def delete_resource(url):
"""
Runs a DELETE request on any Delete endpoint in the Zendesk API
:param url: A full endpoint url, such as 'https://support.zendesk.com/api/v2/help_center/articles/2342572.json'
:return: If successful, a 204 status code. If not, None
"""
response = requests.delete(url, auth=get_auth())
if response.status_code == 429:
print('Rate limited! Please wait.')
time.sleep(int(response.headers['retry-after']))
response = requests.delete(url, auth=get_auth())
if response.status_code != 204:
print('Failed to delete record with error {}'.format(response.status_code))
print(response.text)
return False
return None
def generate_headers(request, method, url):
code = 200
content_type = "text/html"
if method not in ('GET', 'POST'):
code = 405
elif re.match(r'^/static', url):
if os.path.isfile(url[1:]):
subp = url[8:]
content_type = get_content_type(subp)
else:
code = 404
else:
if re.match(r'^/json', url):
content_type = 'application/json'
if method == 'GET' and url not in URLS_GET and url not in URLS_GET_ajax:
code = 404
elif method == 'POST' and url not in URLS_POST:
code = 404
code2 = get_auth(request)
code = code2 if code == 200 else code
request.response.code = code
request.response.content_type = content_type
return code
def __init__(self, db_config,
measure_timestamp_diff=False,
blocking_auth=True,
compression=None,
verbose=10):
guest = db_config.get('guest')
self.app = pyrebase.initialize_app(db_config)
if compression is None:
compression = db_config.get('compression')
self.auth = None
if not guest and 'serviceAccount' not in db_config.keys():
self.auth = get_auth(self.app,
db_config.get("use_email_auth"),
db_config.get("email"),
db_config.get("password"),
blocking_auth)
self.logger = logs.getLogger('FirebaseArtifactStore')
self.logger.setLevel(verbose)
super(FirebaseArtifactStore, self).__init__(
measure_timestamp_diff,
compression=compression)
def get_resource(url):
"""
Returns a single HC resource
:param url: A full endpoint url, such as 'https://support.zendesk.com/api/v2/help_center/articles/2342572.json'
:return: Dict of a resource, or False if the request failed.
"""
resource = None
response = requests.get(url, auth=get_auth())
if response.status_code == 429:
print('Rate limited! Please wait.')
time.sleep(int(response.headers['retry-after']))
response = requests.get(url, auth=get_auth())
if response.status_code != 200:
print('Failed to get record with error {}:'.format(response.status_code))
print(response.text)
return False
for k, v in response.json().items():
resource = v
if type(resource) is dict:
return resource
return None
def retrieve_data_gen(username,
password,
template,
query_args=None,
single_request=False):
auth = get_auth(username, password)
query_args = get_query_args(query_args)
logger.info(f"The auth for the user is {auth}")
per_page = 100
page = 0
while True:
page = page + 1
request = construct_request(per_page, page, template, auth) # noqa
r, errors = get_response(request, auth, template)
status_code = int(r.getcode())
retries = 0
while retries < 3 and status_code == 502:
print(
'API request returned HTTP 502: Bad Gateway. Retrying in 5 seconds'
)
retries += 1
time.sleep(5)
request = construct_request(per_page, page, query_args, template,
auth) # noqa
r, errors = get_response(request, auth, template)
status_code = int(r.getcode())
if status_code != 200:
template = 'API request returned HTTP {0}: {1}'
errors.append(template.format(status_code, r.reason))
logger.error(errors)
response = json.loads(r.read().decode('utf-8'))
if len(errors) == 0:
if type(response) == list:
for resp in response:
yield resp
if len(response) < per_page:
break
elif type(response) == dict and single_request:
yield response
if len(errors) > 0:
logger.error(errors)
if single_request:
break
def configure():
get_event_loop().run_until_complete(gsm.at_connect(PROXY_IP, USERNAME, PASSWORD))
auth = get_auth(gsm.CNUM, "", b'26485001')
get_event_loop().run_until_complete(gsm.start_http_session(PROXY_IP))
server_nonce = yield from gsm.http_action("GET", url(server_url, port, "conf"), time=timestamp(), auth=auth)
get_event_loop().run_until_complete(gsm.terminate_http_session())
auth = get_auth(gsm.CNUM, server_nonce, b'42849900')
get_event_loop().run_until_complete(gsm.start_http_session(PROXY_IP))
conf = yield from gsm.http_action("GET", url(server_url, port, "conf"), time=timestamp(), auth=auth)
get_event_loop().run_until_complete(gsm.terminate_http_session())
get_event_loop().run_until_complete(gsm.at_disconnect())
# Parse configuration settings
if conf:
track_location = parse_conf("track_location", conf)
track_state = parse_conf("track_state", conf)
measure_ms = parse_conf("measure_interval", conf)
stream_ms = parse_conf("stream_interval", conf)
fixed_state_track = parse_conf("fixed_state_track", conf)
verbose = parse_conf("verbose", conf)
def stream(stream_ms):
while True:
yield from sleep(stream_ms)
print("Authorizing...")
auth = get_auth(gsm.CNUM, "", b'26485001')
get_event_loop().call_soon(gsm.at_connect(PROXY_IP, USERNAME, PASSWORD))
print("Preauth: %s" %auth)
print("Initializing HTTP session...")
get_event_loop().call_soon(gsm.start_http_session(PROXY_IP))
print("Retrieving server nonce...")
get_event_loop().call_soon(gsm.start_http_session(PROXY_IP))
server_nonce = yield from gsm.http_action("GET", url(server_url, port, "stream"), time=timestamp(), auth=auth)
print("Possible server nonce: %s" %server_nonce[:])
if server_nonce and not "CME" in server_nonce:
print("Succesfully obtained server nonce %s" %server_nonce)
auth = get_auth(gsm.CNUM, server_nonce, b'42849900')
else:
print("Unable to obtain server nonce")
print("Stopping.")
get_event_loop().call_soon(gsm.terminate_http_session())
print("Running production authorization.")
coords = "test"
state = "test"
data = '{"auth":%s, %s, ' %(auth, coords)
if track_state:
data += '"state":%s, ' %state
data += '"time":%s}' %(timestamp())
print(data)
print("Posting location data...")
get_event_loop().call_soon(gsm.start_http_session(PROXY_IP))
post = yield from gsm.http_action("POST", url(server_url, port, "stream"), data=data)
get_event_loop().call_soon(gsm.terminate_http_session())
get_event_loop().call_soon(gsm.at_disconnect())
def configure(self):
""" Daily configuration call. gsm.at_id() must be performed, as well as at_connect """
gsm.at_connect(self.PROXY_IP, self.USERNAME, self.PASSWORD)
auth = get_auth(gsm.CNUM, "", b'26485001') # Retrieve a server nonce
# urc add event "CONNECT OK"
self.at_start_http_session()
server_nonce = yield from self.http_action("GET", self.url("conf"), time=self.timestamp(), auth=auth) # Retrieve servernonce
auth = get_auth(gsm.CNUM, server_nonce, b'42849900')
# urc event del and if everything worked out fine,
# Obtain configuration
conf = yield from self.http_action("GET", self.url("conf"), time=self.timestamp(), auth=auth)
self.at_terminate_http_session() # TODO: folded with individual requests or like this?
self.at_disconnect()
self.track_location = parse_conf("track_location", conf)
self.track_state = parse_conf("track_state", conf)
self.measure_ms = parse_conf("measure_interval", conf)
self.stream_ms = parse_conf("stream_interval", conf)
self.fixed_state_track = = parse_conf("fixed_state_track", conf)
self.verbose = parse_conf("verbose", conf)
def post_resource(url, data, status=201):
"""
:param url:
:param data:
:param status: HTTP status. Normally 201 but some POST requests return 200
:return: Python data, or False if the request failed.
"""
resource = None
headers = {'Content-Type': 'application/json'}
subdomain = urlparse(url).hostname.split('.')[0]
response = requests.post(url, json=data, auth=get_auth(subdomain), headers=headers)
if response.status_code == 429:
print('Rate limited! Please wait.')
time.sleep(int(response.headers['retry-after']))
response = requests.post(url, json=data, auth=get_auth(), headers=headers)
if response.status_code != status:
print('Failed to create record with error {}:'.format(response.status_code))
print(response.text)
return False
for k, v in response.json().items():
resource = v
if type(resource) is dict:
return resource
return None
def get_github_repo_url(username, password, repository, prefer_ssh=True):
# if args.prefer_ssh:
# return repository['ssh_url']
if repository.get('is_gist'):
return repository['git_pull_url']
if prefer_ssh:
return repository['ssh_url']
##if its a private url
auth = get_auth(username, password, False)
if auth:
logger.info(f"Auth is prsent {auth}")
repo_url = 'https://{0}@{1}/{2}/{3}.git'.format(
auth, get_github_host(), repository['owner']['login'],
repository['name'])
else:
repo_url = repository['clone_url']
return repo_url
def __init__(self,
config,
verbose=10,
blocking_auth=True,
compression=None):
# TODO: implement connection
self.url = config.get('serverUrl')
self.verbose = verbose
self.logger = logs.getLogger('HTTPProvider')
self.logger.setLevel(self.verbose)
self.auth = None
self.app = pyrebase.initialize_app(config)
guest = config.get('guest')
if not guest and 'serviceAccount' not in config.keys():
self.auth = get_auth(self.app, config.get("use_email_auth"),
config.get("email"), config.get("password"),
blocking_auth)
self.compression = compression
if self.compression is None:
self.compression = config.get('compression')
def get_results_page():
q = request.args.get('search')
tweets = search(q, 10)
#Mongo Database
class MyStreamListener(StreamListener):
# Class that Tweepy has built, has collects all functionality of tweets
def __init__(self):
super(MyStreamListener, self).__init__()
self.num_tweets = 0
def on_data(self, data):
if self.num_tweets < 10:
self.num_tweets += 1
try:
with MongoClient(MONGODB_URI) as conn:
db = conn[MONGODB_NAME]
coll = db[q]
coll.insert(json.loads(data))
return True
except BaseException as e:
print ("Failed on_data: %s" % str(e))
return True
else:
return False
def on_error(self, status):
print(status)
return True
auth = get_auth()
twitter_stream = Stream(auth, MyStreamListener())
twitter_stream.filter(track=q)
return render_template("results.html", tweets = tweets, q = q)
def __init__(
self,
db_config,
blocking_auth=True,
verbose=10,
store=None,
compression=None):
guest = db_config.get('guest')
self.app = pyrebase.initialize_app(db_config)
self.logger = logs.getLogger(self.__class__.__name__)
self.logger.setLevel(verbose)
self.compression = compression
if self.compression is None:
self.compression = db_config.get('compression')
self.auth = None
if not guest and 'serviceAccount' not in db_config.keys():
self.auth = get_auth(self.app,
db_config.get("use_email_auth"),
db_config.get("email"),
db_config.get("password"),
blocking_auth)
self.store = store if store else FirebaseArtifactStore(
db_config,
verbose=verbose,
blocking_auth=blocking_auth,
compression=self.compression
)
if self.auth and not self.auth.expired:
self.register_user(None, self.auth.get_user_email())
self.max_keys = db_config.get('max_keys', 100)
def lyrics(bot, update, user_data):
user_id = str(update.message.chat_id) + '_' + update.message.from_user['username']
logging.info("Starting lyrics for user_id: {}".format(user_id))
sp_oauth = auth.get_auth(username=user_id, scope=scope)
token_info = sp_oauth.get_cached_token()
# If there is no cached token, ask for it
if not token_info:
reply_text = "Token unavaliable. Please visit {} and paste/share the url you obtain (https://localhost...).".format(sp_oauth.get_authorize_url())
update.message.reply_text(reply_text)
user_data['sp_oauth'] = sp_oauth
logging.info("text: %s, reply: %s", update.message.text, reply_text)
return TYPING_URL
else:
token = token_info['access_token']
current = get_currently_playing(token)
if not current:
reply_text = "No song currently playing."
update.message.reply_text(reply_text)
logging.info("text: %s, reply: %s", update.message.text, reply_text)
return ConversationHandler.END
item = current['item']
if not item:
reply_text = "No song detected. Perhaps it's a podcast?"
update.message.reply_text(reply_text)
logging.info("text: %s, reply: %s", update.message.text, reply_text)
return ConversationHandler.END
name = item['name']
artist = item['artists'][0]['name']
l = get_lyrics_genius(song=name, artist=artist)
if l:
reply_text = "Song: {}\nArtist: {}\n{}".format(name, artist, l)
else:
reply_text = "Sorry, no lyrics found for song: {}, artist: {}".format(name, artist)
update.message.reply_text(reply_text)
logging.info("text: %s, reply: %s", update.message.text, reply_text)
return ConversationHandler.END
def __init__(self):
self.auth = auth.get_auth()
self.commands = commands
self.client = BotClient(partial(self.handle_message))
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from auth import get_auth
import json
q = "Atlas Shrugged"
count = 100
twitter_api = get_auth()
search_results = twitter_api.search.tweets(q=q, count=count)
statuses = search_results['statuses']
for _ in range(5):
print("Length of statuses", len(statuses))
next_results = ""
try:
next_results = search_results['search_metadata']['next_results']
print("got next results")
except KeyError as e:
print("no more results")
break
kwargs = dict([kv.split('=') for kv in next_results[1:].split("&")])
#print( "next_results: ", next_results )
#print( "kwargs: ", kwargs )
def post_attachment(url, attachment, status=201):
resource = None
file = get_resource(attachment['content_url'], attachment=True)
subdomain = urlparse(url).hostname.split('.')[0]
file_path = os.path.join('attachments', attachment['file_name'])
with open(file_path, 'wb') as f:
f.write(file)
with open(file_path, 'rb') as u:
response = requests.post(url, data={'inline': attachment['inline']}, files={'file':u}, auth=get_auth(subdomain))
if response.status_code == 429:
print('Rate limited! Please wait.')
time.sleep(int(response.headers['retry-after']))
response = requests.post(url, files={'inline':attachment['inline'], 'file':u})
if response.status_code != status:
print('Failed to create record with error {}:'.format(response.status_code))
print(response.text)
return False
for k, v in response.json().items():
resource = v
if type(resource) is dict:
return resource
return None
from aip import AipFace
import base64, auth, json, urllib, urllib2
APP_ID = '10440639'
API_KEY = 'REeeig5V9NpTkffDiqvTSyuV'
SECRET_KEY = 'oUdGrgbktr8G5BpirKwnqMdscGlGV3wu'
client = AipFace(APP_ID, API_KEY, SECRET_KEY)
content = auth.get_auth()
content = eval(content)
print content['access_token']
request_url = 'https://aip.baidubce.com/rest/2.0/face/v1/detect'
f = open('face.jpg', 'rb')
img = base64.b64encode(f.read())
print img
params = {
"face_fields":
"age,beauty,expression,faceshape,gender,glasses,landmark,race,qualities",
"image": img,
"max_face_num": 5
}
params = urllib.urlencode(params)
request_url = request_url + "?access_token=" + content['access_token']
request = urllib2.Request(url=request_url, data=params)
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
response = urllib2.urlopen(request)
result = eval(response.read())
print('age:%s' % result['result'][0]['age'])
print('beauty:%s' % result['result'][0]['beauty'])
print('gender:%s' % result['result'][0]['gender'])
import requests
from auth import get_auth
MY_URL_NUM = '2361076'
AUTH = get_auth()
BASE_URL = 'https://basecamp.com/{}/api/v1/'
MY_URL = BASE_URL.format(MY_URL_NUM)
class BCError(Exception):
pass
class BaseCampConnectionError(BCError):
pass
class BaseCampConnectionAuthError(BCError):
pass
class BaseCamp(object):
BASE_URL = 'https://basecamp.com/{}/api/v1/'
def __init__(self, url_num, user=None, pw=None):
self.base_url = BaseCamp.BASE_URL.format(url_num)
if user is not None and pw is not None:
self.auth = (user, pw)
else:
self.auth = ''
def main():
while True:
handler()
time.sleep(get_auth('monitor_interval'))
def stream_tweets(query, count):
auth = get_auth()
tweets = []
twitter_stream = Stream(auth, MyStreamListener(count, tweets))
twitter_stream.filter(track=[query])
return tweets
import requests
from auth import get_auth
MY_URL_NUM = '2361076'
AUTH = get_auth()
BASE_URL = 'https://basecamp.com/{}/api/v1/'
MY_URL = BASE_URL.format(MY_URL_NUM)
class BCError(Exception):
pass
class BaseCampConnectionError(BCError):
pass
class BaseCampConnectionAuthError(BCError):
pass
class BaseCamp(object):
BASE_URL = 'https://basecamp.com/{}/api/v1/'
def __init__(self,url_num,user=None,pw=None):
self.base_url = BaseCamp.BASE_URL.format(url_num)
if user is not None and pw is not None:
self.auth = (user,pw)
else:
self.auth = ''
def connect(self):
if not self.auth:
raise BaseCampConnectionAuthError
#!/usr/bin/env python3
import json
import os
import sys
import time
import requests
import auth
article_limit = None
if len(sys.argv) > 1:
article_limit = sys.argv[1]
rootDir = 'html'
for dirName, subdirList, fileList in os.walk(rootDir):
for fname in fileList:
article, extension = os.path.splitext(fname)
if extension == '.html':
if article_limit is None or article_limit == article:
url = 'https://memsource.zendesk.com/api/v2/help_center/articles/{}/translations/en-us.json'.format(
article)
print(f"Requesting {url}")
headers = {'Content-Type': 'application/json'}
f = open(f"{dirName}/{fname}", "r")
data = {"translation": {"body": f.read()}}
response = requests.put(url, auth=auth.get_auth(), headers=headers, data=json.dumps(data))
print(response)
time.sleep(0.1)
import os
import requests
import auth
# ----- Variables ----- #
auth = auth.get_auth()
loader_path = os.path.join('maps', 'loader.json')
unloader_path = os.path.join('maps', 'unloader.json')
global_map_path = os.path.join('maps', 'global_map.json')
# ----- Functions ----- #
def get_translation(article_id, hc, locale='en-us'):
"""
:param article_id: Help Center article id
:param hc: Help Center subdomain. Example: mondocam in mondocam.zendesk.com/hc
:param locale: The article locale
:return: A JSON object with the translation properties
"""
url = 'https://{}.zendesk.com/api/v2/help_center/articles/{}/translations/{}.json'.format(
hc, article_id, locale)
response = requests.get(url, auth=auth)
if response.status_code == 200:
return response.json()['translation']
else:
print('Failed to get article {} with error {}'.format(
article_id, response.status_code))
return None
class MyStreamListener(StreamListener):
def __init__(self):
super(MyStreamListener, self).__init__()
self.num_tweets = 0
def on_data(self, data):
if self.num_tweets < limit:
self.num_tweets += 1
try:
with open('new_tweet_mining.json', 'a') as tweet_file:
tweet_file.write(data)
return True
except BaseException as e:
print ("Failed on_data: %s" % str(e))
return True
else:
return False
def on_error(self, status):
print(status)
return True
auth = get_auth()
twitter_stream = Stream(auth, MyStreamListener())
twitter_stream.filter(track=keyword_list)
limit = 1000
filename = "repealNo.json"
class MyStreamListener(StreamListener):
def __init__(self):
super(MyStreamListener, self).__init__()
self.num_tweets = 0
def on_data(self, data):
if self.num_tweets < limit:
self.num_tweets += 1
try:
with open(filename, 'a') as tweet_file:
tweet_file.write(data)
return True
except BaseException as e:
print("Failed on_data: %s" % str(e))
return True
else:
return False
def on_error(self, status):
print(status)
return True
auth = get_auth()
twitter_stream = Stream(auth, MyStreamListener())
twitter_stream.filter(track=keyword_list)
import json
import time
import pathlib
import auth
json_props = ['html_url', 'title']
page = 1
while True:
url = 'https://memsource.zendesk.com/api/v2/help_center/en-us/articles.json?page={}'.format(
page)
print(f"Requesting {url}")
headers = {'Content-Type': 'application/json'}
response = requests.get(url, auth=auth.get_auth(), headers=headers)
data = json.loads(response.text)
if len(data['articles']) == 0:
break
for article in data["articles"]:
dirname = 'html/{}/{}'.format(article['section_id'], article['id'])
pathlib.Path(dirname).mkdir(parents=True, exist_ok=True)
html_filename = '{}/{}.html'.format(dirname, article['id'])
json_filename = '{}/{}.json'.format(dirname, article['id'])
print(f"Writing {html_filename}")
f = open(html_filename, 'w')
body = article['body'] if article['body'] else ""
f.write(body)
