def desktop_app():
"""Emulation of desktop app.
Your keys should be created with project type "Desktop".
Returns: ``upwork.Client`` instance ready to work.
"""
print 'Emulating desktop app'
client = upwork.Client(public_key, secret_key)
auth_url = client.auth.get_authorize_url()
verifier = raw_input('Please enter the verification code you get '
'following this link:\n{0}\n\n> '.format(auth_url))
print 'Retrieving keys.... '
access_token, access_token_secret = client.auth.get_access_token(verifier)
print 'OK'
# For further use you can store ``access_toket`` and
# ``access_token_secret`` somewhere
client = upwork.Client(public_key,
secret_key,
oauth_access_token=access_token,
oauth_access_token_secret=access_token_secret)
return client
def get_upwork_client():
"""This function sets up a client-server
connection and keeps it for a day."""
cred_file = open('credentials.json', 'r')
cred_json = json.load(cred_file)
cred_file.close()
public_key = cred_json['public_key']
secret_key = cred_json['secret_key']
upwork_client = upwork.Client(public_key, secret_key)
auth_url = upwork_client.auth.get_authorize_url()
webbrowser.open(url=auth_url, autoraise=True, new=2)
verifier = input(
'Please enter the verification code you get '
'following this link:\n{0}\n\n> '.format(auth_url))
access_token, access_token_secret = \
upwork_client.auth.get_access_token(verifier)
upwork_client = upwork.Client(
public_key, secret_key,
oauth_access_token=access_token,
oauth_access_token_secret=access_token_secret)
return upwork_client
def web_based_app():
"""Emulation of web-based app.
Your keys should be created with project type "Web".
Returns: ``upwork.Client`` instance ready to work.
"""
print("Emulating web-based app")
public_key = input('Please enter public key: > ')
secret_key = input('Please enter secret key: > ')
#Instantiating a client without an auth token
client = upwork.Client(public_key, secret_key)
print("Please to this URL (authorize the app if necessary):")
print(client.auth.get_authorize_url())
print("After that you should be redirected back to your app URL with " + \
"additional ?oauth_verifier= parameter")
verifier = input('Enter oauth_verifier: ')
oauth_access_token, oauth_access_token_secret = \
client.auth.get_access_token(verifier)
# Instantiating a new client, now with a token.
# Not strictly necessary here (could just set `client.oauth_access_token`
# and `client.oauth_access_token_secret`), but typical for web apps,
# which wouldn't probably keep client instances between requests
client = upwork.Client(public_key,
secret_key,
oauth_access_token=oauth_access_token,
oauth_access_token_secret=oauth_access_token_secret)
return client
def update_job(id):
json_data = request.args.to_dict()
try:
access_token = session['access_token']
access_token_secret = session['access_token_secret']
client = upwork.Client(public_key=credentials.public_key,
secret_key=credentials.secret_key,
oauth_access_token=access_token,
oauth_access_token_secret=access_token_secret,
timeout=30)
user_info = session.get('user_info')
team_id = user_info['teams'][0]['reference']
result = client.hr.update_job(
job_id=id,
buyer_team_reference=team_id,
title=json_data.get("title", ""),
description=json_data.get("snippet", ""),
visibility=json_data.get("visibility", ""),
category2="Data Science & Analytics",
subcategory2=json_data.get("subcategory2", ""),
start_date=json_data.get("start_date", ""),
budget=json_data.get("budget", 10),
duration=json_data.get("duration", 10),
status='open')
return jsonify(result=result['job']['message'])
except Exception as e:
print "Exception: {}".format(e)
return None
def request_access_token():
""" Request an Access Token and Access Token Secret """
consumer_key, consumer_secret = load_api_key()
client_config = upwork.Config({
'consumer_key': consumer_key,
'consumer_secret': consumer_secret
})
client = upwork.Client(client_config)
try:
client_config.access_token
client_config.access_token_secret
except AttributeError:
verifier = input(
f'Please enter the verification code you get '
f'following this link:\n{client.get_authorization_url()}\n\n>'
)
print('Retrieving keys.... ')
# Once you receive the request token and the resource owner's authorization
# (verifier code), you are ready to request Upwork Server an Access token
access_token, access_token_secret = client.get_access_token(verifier)
print('OK')
with open(config.ACCESS_TOKEN_FILENAME, "w") as f:
f.write(access_token+"\n")
f.write(access_token_secret)
return [access_token, access_token_secret]
def start():
try:
access_token = session['access_token']
access_token_secret = session['access_token_secret']
client = upwork.Client(public_key=credentials.public_key,
secret_key=credentials.secret_key,
oauth_access_token=access_token,
oauth_access_token_secret=access_token_secret,
timeout=30)
user_info = session.get('user_info')
if not user_info:
user_info = client.auth.get_info()
session['user_info'] = user_info
user_info['teams'] = client.hr.get_teams(
) # get this info everytime, might have changed
# pretty_print(user_info)
first_name = user_info['auth_user']['first_name']
last_name = user_info['auth_user']['last_name']
profile_pic = user_info['info']['portrait_32_img']
jobs_list = get_jobs(client, user_info['teams'])
# print "### Jobs:"
# pretty_print(jobs_list)
is_client = user_info['info']['capacity']['buyer'] == 'yes'
return render_template("index.html",
first_name=first_name,
last_name=last_name,
profile_pic=profile_pic,
jobs_list=jobs_list,
is_client=is_client)
except Exception as err:
print err
session.clear()
return render_template("login.html")
def saved_access_client(self, token_file):
tokens = token_file.read().split("\n")
oauth_access_token, oauth_access_token_secret = tokens[0], tokens[1]
client = upwork.Client(
self.public_key,
self.secret_key,
oauth_access_token=oauth_access_token,
oauth_access_token_secret=oauth_access_token_secret)
return client
def login():
client = upwork.Client(public_key=credentials.public_key,
secret_key=credentials.secret_key,
timeout=30)
request_token, request_token_secret = client.auth.get_request_token()
session['request_token'] = request_token
session['request_token_secret'] = request_token_secret
authorize_url = client.auth.get_authorize_url(
callback_url=request.base_url + "/upwork")
return redirect(authorize_url)
def create_new_client(self, token_file):
client = upwork.Client(self.public_key, self.secret_key)
print "Please to this URL (authorize the app if necessary):"
print client.auth.get_authorize_url()
print "After that you should be redirected back to your app URL with " + \
"additional ?oauth_verifier= parameter"
verifier = raw_input('Enter oauth_verifier: ')
oauth_access_token, oauth_access_token_secret = \
client.auth.get_access_token(verifier)
token_file.write(oauth_access_token + "\n" + oauth_access_token_secret)
client = upwork.Client(
self.public_key,
self.secret_key,
oauth_access_token=oauth_access_token,
oauth_access_token_secret=oauth_access_token_secret)
return client
def __client(self):
'''
Authenticate to Upwork API
:return: upwork client obj
'''
try:
upwork_client = upwork.Client(self.public_key, self.secret_key)
verifier = upwork_client.auth.get_authorize_url()
oauth_access_token, oauth_access_token_secret = \
upwork_client.auth.get_access_token(verifier)
client = upwork.Client(
self.public_key,
self.secret_key,
oauth_access_token=oauth_access_token,
oauth_access_token_secret=oauth_access_token_secret)
except Exception as e:
print("Error: unable to authenticate ", e)
raise
return client
def login_callback():
oauth_token = request.args.get('oauth_token', None)
oauth_verifier = request.args.get('oauth_verifier', None)
client = upwork.Client(public_key=credentials.public_key,
secret_key=credentials.secret_key,
timeout=30)
client.auth.request_token = session['request_token']
client.auth.request_token_secret = session['request_token_secret']
session.clear()
access_token, access_token_secret = client.auth.get_access_token(
oauth_verifier)
session['access_token'] = access_token
session['access_token_secret'] = access_token_secret
return redirect("/")
def get_client():
cred_file = open('credentials.json', 'r')
cred_json = json.load(cred_file)
cred_file.close()
public_key = cred_json['public_key']
secret_key = cred_json['secret_key']
upwork_client = upwork.Client(public_key, secret_key)
auth_url = upwork_client.auth.get_authorize_url()
# Opens a new tab in default web browser
webbrowser.open(url=auth_url, autoraise=True, new=2)
print 'Go to the mentioned URL : {}'.format(auth_url)
verifier = raw_input('Enter Verifier: ')
(token, token_secret) = upwork_client.auth.get_access_token(verifier)
upwork_client = upwork.Client(public_key,
secret_key,
oauth_access_token=token,
oauth_access_token_secret=token_secret)
return upwork_client
def get_tokens():
with open('keys.json', 'r') as json_file:
config = json.load(json_file)
client = upwork.Client(upwork.Config(config))
token = client.get_request_token()
print(token)
verifier = input('Please enter the verification code you get '
'following this link:\n{0}\n\n> '.format(
client.get_authorization_url()))
access_token, access_token_secret = client.get_access_token(verifier)
print(access_token, access_token_secret)
def get_client(self):
"""Emulation of desktop app.
Your keys should be created with project type "Desktop".
Returns: ``odesk.Client`` instance ready to work.
"""
client = upwork.Client(PUBLIC_KEY, SECRET_KEY)
url = client.auth.get_authorize_url()
if USERNAME and PASSWORD:
try:
browser = create_browser()
verifier = get_verifier(url, browser)
except Exception, e:
raise e
finally:
def get_desktop_client():
"""Emulation of desktop app.
Your keys should be created with project type "Desktop".
Returns: ``upwork.Client`` instance ready to work.
"""
print("Emulating desktop app")
consumer_key = input("Please enter consumer key: > ")
consumer_secret = input("Please enter key secret: > ")
config = upwork.Config({
"client_id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
"client_secret": "xxxxxxxxxxxxx",
"redirect_uri": "https://a.callback.url",
})
# If token data already known and saved, you can reuse them
# by adding 'token' parameter to the config
# token = {'access_token': 'xxxxxxxxxxxxxxxxxx', 'expires_at': 1590479276.547947, 'expires_in': '86400', 'refresh_token': 'xxxxxxxxxxxxxxxxxxxxxxxx', 'token_type': 'Bearer'}
# config = upwork.Config({'client_id': 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', 'client_secret': 'xxxxxxxxxxxxx', 'token': token})
client = upwork.Client(config)
try:
config.token
except AttributeError:
authorization_url, state = client.get_authorization_url()
# cover "state" flow if needed
authz_code = input(
"Please enter the full callback URL you get "
"following this link:\n{0}\n\n> ".format(authorization_url))
print("Retrieving access and refresh tokens.... ")
token = client.get_access_token(authz_code)
# WARNING: the access token will be refreshed automatically for you
# in case it's expired, i.e. expires_at < time(). Make sure you replace the
# old token accordingly in your security storage. Call client.get_actual_config
# periodically to sync-up the data
pprint(token)
print("OK")
# For further use you can store ``token`` data somewhere
return client
def __init__(self):
self.log = open('json_files/log_upwork_data_collection_2017_10_23_unitedstates_webdev.txt', 'a') # Creating a log
self.log.write("We have started collecting data!" + "\n")
# Connect to the database
self.conn = psycopg2.connect("dbname=eureka01")
self.cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json)
# Instantiating a new client with access token
self.public_key = 'ed8ff5784819b4dbadb800d8e9e10e21'
self.secret_key = '03a6e678f4ee3246'
self.oauth_access_token = 'b60900600fc9e6da57d7711c2d098591'
self.oauth_access_token_secret = '5022f67f9bccb73d'
self.client = upwork.Client(self.public_key, self.secret_key,
oauth_access_token=self.oauth_access_token,
oauth_access_token_secret=self.oauth_access_token_secret)
def get_api_token():
client = upwork.Client(os.environ["upkey"], os.environ["upsecret"])
authorize_url = client.auth.get_authorize_url()
doc = Node(authorize_url, "url", "chrome")
print("Navigating authorize url...")
doc.select("#login_username").send_keys(os.getenv("email"))
doc.select("#login_password").send_keys(os.getenv("up") + "#u")
doc.select("#layout form").submit()
print("Navigating token url...")
verifier_el = doc.select("#main > div")
verifier = first_match("(?<=oauth_verifier=).+", verifier_el.text)
oauth_token, oauth_token_secret = client.auth.get_token(verifier)
oauth_token = oauth_token.decode("utf-8")
oauth_token_secret = oauth_token_secret.decode("utf-8")
print(oauth_token, oauth_token_secret)
return oauth_token, oauth_token_secret
def get_client():
"""Emulation of desktop app.
Your keys should be created with project type "Desktop".
Returns: ``upwork.Client`` instance ready to work.
"""
print "Emulating desktop app"
public_key = PUBLIC_KEY
secret_key = SECRET_KEY
client = upwork.Client(public_key, secret_key)
url = client.auth.get_authorize_url()
if USERNAME and PASSWORD:
try:
browser = create_browser()
verifier = get_verifier(url, browser)
except Exception, e:
raise e
finally:
def job_existing(id):
try:
access_token = session['access_token']
access_token_secret = session['access_token_secret']
client = upwork.Client(public_key=credentials.public_key,
secret_key=credentials.secret_key,
oauth_access_token=access_token,
oauth_access_token_secret=access_token_secret,
timeout=30)
update_job_info = client.hr.get_job(id)
pretty_print_dict(update_job_info)
update_job_info["job_id"] = id
if update_job_info["category2"] != "Data Science & Analytics":
return job(client=client,
warning="The selected job is not a Data Science job!")
return job(client=client, update_job_info=update_job_info)
except Exception as err:
print err
session.clear()
return render_template("login.html")
def job(client=None, update_job_info=None, warning=None):
try:
if not client:
access_token = session['access_token']
access_token_secret = session['access_token_secret']
client = upwork.Client(
public_key=credentials.public_key,
secret_key=credentials.secret_key,
oauth_access_token=access_token,
oauth_access_token_secret=access_token_secret,
timeout=30)
user_info = session.get('user_info')
profile_pic = user_info['info']['portrait_32_img']
is_client = user_info['info']['capacity']['buyer'] == 'yes'
if not GLOBAL_VARIABLE.has_key("skills_list"):
skills_list = client.provider.get_skills_metadata()
print "### Skills list:"
# pretty_print(skills_list)
skills_list_js = '['
for skill in skills_list:
skills_list_js = skills_list_js + '"' + str(skill) + '",'
skills_list_js = skills_list_js[0:-1] + ']'
print skills_list_js
GLOBAL_VARIABLE["skills_list"] = skills_list_js
client_info = get_client_data(client)
return render_template(
"job.html",
profile_pic=profile_pic,
current_date=datetime.date.today().strftime("%m-%d-%Y"),
skills_list=GLOBAL_VARIABLE["skills_list"],
client_info=client_info,
update_job_info=update_job_info,
warning=warning,
is_client=is_client)
except Exception as err:
print err
session.clear()
return render_template("login.html")
def get_desktop_client():
"""Emulation of a desktop application.
Your key should be created with the project type "Desktop".
Returns: ``upwork.Client`` instance ready to work.
"""
print("Emulating desktop app")
consumer_key = input('Please enter consumer key: > ')
consumer_secret = input('Please enter key secret: > ')
config = upwork.Config({
'consumer_key': consumer_key,
'consumer_secret': consumer_secret
})
"""Assign access_token and access_token_secret if they are known
config = upwork.Config({\
'consumer_key': 'xxxxxxxxxxx',\
'consumer_secret': 'xxxxxxxxxxx',\
'access_token': 'xxxxxxxxxxx',\
'access_token_secret': 'xxxxxxxxxxx'})
"""
client = upwork.Client(config)
try:
config.access_token
config.access_token_secret
except AttributeError:
verifier = input('Please enter the verification code you get '
'following this link:\n{0}\n\n> '.format(
client.get_authorization_url()))
print('Retrieving keys.... ')
access_token, access_token_secret = client.get_access_token(verifier)
print('OK')
# For further use you can store ``access_toket`` and
# ``access_token_secret`` somewhere
return client
def fetch_team_report(self, from_date, to_date):
client = upwork.Client(
os.environ['UPWORK_PUBLIC_KEY'],
os.environ['UPWORK_SECRET_KEY'],
oauth_access_token=os.environ['UPWORK_OAUTH_TOKEN'],
oauth_access_token_secret=os.environ['UPWORK_OAUTH_TOKEN_SECRET'],
)
pattern = '%Y-%m-%d'
fd = from_date.strftime(pattern)
td = to_date.strftime(pattern)
query = (
"SELECT worked_on, provider_id, provider_name, sum(hours), memo "
"WHERE worked_on >= '{}' AND worked_on <= '{}'".format(fd, td))
report = client.timereport.get_team_report(settings.UPWORK_COMPANY_ID,
settings.UPWORK_TEAM_ID,
query)
if report.get('status', 'success') == 'success':
return report
else:
raise RuntimeError(report)
def download():
try:
api_key, api_key_secret = load_api_key()
except CredentialsNotFoundError as e:
import os
return jsonify({'msg': os.listdir("./data")})
access_token, access_token_secret = load_access_token()
client_config = upwork.Config({
'consumer_key': api_key,
'consumer_secret': api_key_secret,
'access_token': access_token,
'access_token_secret': access_token_secret
})
client = upwork.Client(client_config)
jobs = search_jobs(client, SEARCH_TERMS)
add_records(jobs)
return jsonify({'msg': "Done"})
def __init__(self):
self.log = open(
'log_upwork_job_collection_2018_04_03_unitedstates.txt',
'a') # Creating a log
self.log.write("We have started collecting data!" + "\n")
# Connect to the database
self.conn = psycopg2.connect("dbname=eureka01")
self.cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json)
# Instantiating a new client with access token
self.public_key = '956957052307e970d62e6c85e4e52a76'
self.secret_key = '375a5cc1bbe07402'
self.oauth_access_token = '835259548ace3959309d17fa02a7143c'
self.oauth_access_token_secret = '018107b1b8d2bc47'
self.client = upwork.Client(
self.public_key,
self.secret_key,
oauth_access_token=self.oauth_access_token,
oauth_access_token_secret=self.oauth_access_token_secret)
# Control number of pages, size of page, and time interval
size_of_page = 40
number_of_pages = 1
time_interval = 61
logging.basicConfig()
public_key = '956957052307e970d62e6c85e4e52a76'
secret_key = '375a5cc1bbe07402'
oauth_access_token = '835259548ace3959309d17fa02a7143c'
oauth_access_token_secret = '018107b1b8d2bc47'
# Instantiating a new client with access token
client = upwork.Client(public_key,
secret_key,
oauth_access_token=oauth_access_token,
oauth_access_token_secret=oauth_access_token_secret)
# Searching for freelancers
data = {'q': 'united states', 'skills': 'java'}
freelancer_basic_data_list = []
freelancer_detailed_data_list = []
def collect_worker_data():
if collect_worker_data.offset < number_of_pages + 1:
print "We're going to start collecting data now!"
print "The collect_worker_data.offset is"
print collect_worker_data.offset
freelancers_on_page = client.provider_v2.search_providers(
data=data,
verifier = get_verifier(url, browser)
except Exception, e:
raise e
finally:
browser.quit()
else:
verifier = raw_input('Please enter the verification code you get '
'following this link:\n{0}\n\n> '.format(url))
LOGGER.debug('Retrieving keys.... ')
access_token, access_token_secret = client.auth.get_access_token(
verifier)
LOGGER.debug('OK')
client = upwork.Client(PUBLIC_KEY,
SECRET_KEY,
oauth_access_token=access_token,
oauth_access_token_secret=access_token_secret)
return client
def search_jobs(self, *args, **kargs):
LOGGER.debug("search_jobs enter lock")
with self.lock:
try:
LOGGER.debug(threading.currentThread().name +
"search_jobs get lock")
sleep(2)
result = self.client.provider_v2.search_jobs(*args, **kargs)
LOGGER.debug(threading.currentThread().name +
"search_jobs get result")
return result
except Exception as e:
parser = argparse.ArgumentParser()
parser.add_argument("--api-key", help="API Key", required=True)
parser.add_argument("--api-secret", help="API Secret", required=True)
parser.add_argument("--month", help="YYYY-MM format", required=True)
parser.add_argument("--path", help="Reports path", required=True)
parser.add_argument("--access-token", help="Access token", required=False)
parser.add_argument("--access-token-secret",
help="Access token secret",
required=False)
args = parser.parse_args()
if args.access_token is not None and args.access_token_secret is not None:
client = upwork.Client(
upwork.Config({
"consumer_key": args.api_key,
"consumer_secret": args.api_secret,
"access_token": args.access_token,
"access_token_secret": args.access_token_secret,
}))
else:
client = upwork.Client(
upwork.Config({
"consumer_key": args.api_key,
"consumer_secret": args.api_secret
}))
verifier = input("Please enter the verification code you get "
"following this link:\n{0}\n\n> ".format(
client.get_authorization_url()))
access_token, access_token_secret = client.get_access_token(verifier)
print(f"Generated tokens: {access_token}, {access_token_secret}")
client = upwork.Client(
def post(self):
json_data = request.get_json(force=True)
sample_size = 10
days_posted = 365
page_offset = None
if 'sample_size' in json_data:
sample_size = json_data['sample_size']
if sample_size < 1:
return {
'api_name': 'Data module REST API',
'success': False,
'sample-size': 0,
'exception': 'sample_size too small'
}
if 'days_posted' in json_data:
days_posted = json_data['days_posted']
if days_posted < 1:
return {
'api_name':
'Data module REST API',
'success':
False,
'sample-size':
sample_size,
'exception':
'Only non-zero and positive values for days posted allowed'
}
if 'page_offset' in json_data:
page_offset = json_data['page_offset']
if page_offset == "None":
page_offset = None
found_jobs = []
pages = 1 + (sample_size - 1) / max_request_size
print 'pages: ' + str(pages)
_sample_size = max_request_size
exception = "None"
# connect to Upwork
client = upwork.Client(
public_key=credentials.public_key,
secret_key=credentials.secret_key,
oauth_access_token=credentials.oauth_access_token,
oauth_access_token_secret=credentials.oauth_access_token_secret,
timeout=30)
# assemble data in multiple iterations because of maximum number of data we can request
for p in range(0, pages):
if p == pages - 1:
_sample_size = (sample_size % max_request_size) if (
sample_size % max_request_size) != 0 else sample_size
# print 'paging: ' + str(p * max_request_size) + ';' + str(_sample_size)
query_data = {
'q': '*',
'category2': 'Data Science & Analytics',
'job_status': 'completed',
'days_posted': days_posted
}
# try to get data until we either got it or we exceed the limit
for i in range(0, max_tries):
try:
if page_offset is None:
found_jobs.extend(
client.provider_v2.search_jobs(
data=query_data,
page_offset=(p * max_request_size),
page_size=_sample_size))
else:
found_jobs.extend(
client.provider_v2.search_jobs(
data=query_data,
page_offset=page_offset +
(p * max_request_size),
page_size=_sample_size))
print 'Successfully found jobs, page_offset=' + str(
p *
max_request_size) + ', page_size=' + str(_sample_size)
exception = "None"
break
except Exception as e:
print 'Number of tries for job search: ' + str(i)
print e
exception = str(e.code) + ' - ' + e.msg
if found_jobs is not None:
# Uncomment to load from files instead from Upwork
# with open(working_dir + "job_profiles_complete.json") as f:
# job_profiles = json.load(f)
# stored_profiles = {}
# for job_profile in job_profiles:
# stored_profiles[job_profile['id']] = job_profile
#
# with open(working_dir + "found_jobs_4K.json") as f:
# found_jobs = json.load(f)
#
# found_jobs, job_profiles = self.enrich_with_job_profiles(client, found_jobs,
# save_incrementally=False,
# stored_job_profiles=stored_profiles)
found_jobs, job_profiles = self.enrich_with_job_profiles(
client, found_jobs, save_incrementally=True)
with open(
working_dir +
strftime("found_jobs_%d.%m.-%H%M.json", gmtime()),
"a+") as f:
f.truncate()
f.write(json.dumps(found_jobs))
# store the jobs for all the mining modules as fallback
with open(working_dir + "dm_lib/" + JOBS_FILE, "a+") as f:
f.truncate()
f.write(json.dumps(found_jobs))
with open(
working_dir +
strftime("job_profiles_%d.%m.-%H%M.json", gmtime()),
"a+") as f:
f.truncate()
f.write(json.dumps(job_profiles))
response = rdb.table(RDB_JOB_TABLE).insert(found_jobs,
conflict="replace").run(
g.rdb_conn)
if not rdb.table_list().contains(RDB_JOB_OPTIMIZED_TABLE).run(
g.rdb_conn):
rdb.table_create(RDB_JOB_OPTIMIZED_TABLE).run(g.rdb_conn)
else:
rdb.table(RDB_JOB_OPTIMIZED_TABLE).delete().run(g.rdb_conn)
# prepare data with dm_lib
data_frame = prepare_data(file_name='', jobs=found_jobs)
data_frame.date_created = data_frame.date_created.apply(
lambda time: time.to_pydatetime().replace(
tzinfo=rdb.make_timezone("+02:00")))
data_frame['id'] = data_frame.index
rdb.table(RDB_JOB_OPTIMIZED_TABLE).insert(
data_frame.to_dict('records'),
conflict="replace").run(g.rdb_conn)
# make all mining modules aware of the data refresh
update_urls = [
module_url + "update_model/"
for module_url in self.mining_module_urls.values()
]
try:
rs = (grequests.post(u) for u in update_urls)
grequests.map(rs)
except Exception as e:
print "Exception: {}".format(e)
success_criterion = len(
found_jobs) == sample_size and response['inserted'] > 0
if len(found_jobs) != sample_size:
exception = "Only got {} of the requested {} jobs".format(
len(found_jobs), sample_size)
if response['inserted'] <= 0:
exception = "No new samples were returned by the API"
return {
'api_name': 'Data module REST API',
'success': success_criterion,
'sample-size': len(found_jobs),
'exception': exception,
'database-response': response
}
return {
'api_name': 'Data module REST API',
'success': False,
'sample-size': 0,
'exception': exception
}
import upwork
# # url = "https://www.upwork.com/api/profiles/v2/search/jobs.json"
# # POST /api/auth/v1/oauth/token/request
# url = "https://www.upwork.com/api/auth/v1/oauth/token/request"
#
# # response = requests.request("GET", url)
# response = requests.request("POST", url)
#
# print(response.text)
public_key = "6cc64b61b06a6d9446d17daec5aef053"
secret_key = "b923e71b020a22ab"
credentials = {}
#Instantiating a client without an auth token
client = upwork.Client(public_key, secret_key)
# client = upwork.Client(public_key, secret_key, **credentials)
# client.auth.get_authorize_url()
client.auth.get_request_token()
# sleep(10000)
# #example code
#
# client.provider_v2.search_jobs(
# data={PARAMS_DICT}, page_offset=0, page_size=20)
#
# #EXAMPLE REQUEST:
# import upwork
# client = upwork.Client(public_key, secret_key, **credentials)
# data = {'q': 'python', 'title': 'Web developer'}
def get_metadata(client, type):
"""
https://developers.upwork.com/?lang=python#metadata
"""
if type == "categories":
return metadata.Api(client).get_categories_v2()
elif type == "skills":
return metadata.Api(client).get_skills_v2()
elif type == "specialties":
# try client.get("/profiles/v1/metadata/specialties")
return metadata.Api(client).get_specialties()
if __name__ == "__main__":
api_key, api_key_secret = load_api_key()
access_token, access_token_secret = load_access_token()
client_config = upwork.Config({
'consumer_key': api_key,
'consumer_secret': api_key_secret,
'access_token': access_token,
'access_token_secret': access_token_secret
})
client = upwork.Client(client_config)
jobs = search_jobs(client, SEARCH_TERMS)
add_records(jobs)
