def lookup_by_name(self, first_name, last_name):
"""
:return: a tuple of (user_id, is_err)
If user is found, returns (user_id, False)
If user is not found, this is not an error [returns (None, False)]
"""
if not (first_name and last_name):
return response(
400, f'Missing name: "{first_name}" "{last_name}"'), True
try:
result = self.dynamo.query(
ExpressionAttributeValues={
':fn': {
'S': first_name
},
':ln': {
'S': last_name
}
},
KeyConditionExpression='first_name = :fn AND last_name = :ln',
TableName=self.tablename,
IndexName='full_name-index')
items = result['Items']
if len(items) == 0:
return (None, False)
elif len(items) > 1:
return response(
500,
f'More than one person named "{first_name}" "{last_name}"'
), True
else:
item = to_json(items[0])
return item['user_id'], False
except Exception:
raise
def render_project_json(project, tfstate):
if len(tfstate):
role = tfstate['apex_function_role']
if role:
project['role'] = role
project = c.deep_merge(project, {"environment": tfstate})
return c.to_json(project, False)
def render_project_json(project, tfstate):
if len( tfstate ):
role = tfstate['apex_function_role']
if role:
project['role'] = role
project = c.deep_merge(project, {"environment": tfstate})
return c.to_json(project, False)
def get_all(self):
"""
:return: a list of all items. At scale, this should be paginated.
"""
data = self.dynamo.scan(TableName=self.tablename)
items = [to_json(item) for item in data['Items']]
return items
def user_skills(d):
quoted = quote(d[u"handle"])
request = common.make_request(u"/v3.0.0/members/%s/skills/" % quoted)
skills = common.to_json(urllib2.urlopen(request).read())
skills = skills[u"result"][u"content"][u"skills"]
for dd in skills.values():
del dd[u"hidden"]
d[u"skills"] = skills
def user_skills(d):
handle = quote_handle(d[u"handle"])
request = common.make_request(u"/v3/members/%s/skills/" % handle)
raw = common.open_request_and_read(request).decode("utf-8")
skills = common.to_json(raw)
skills = skills[u"result"][u"content"][u"skills"]
for dd in skills.values():
del dd[u"hidden"]
d[u"skills"] = skills
def user_skills(d):
handle = quote_handle(d[u"handle"])
request = common.make_request(u"/v3/members/%s/skills/" % handle)
raw = common.open_request_and_read(request).decode("utf-8")
skills = common.to_json(raw)
skills = skills[u"result"][u"content"][u"skills"]
for dd in skills.values():
del dd[u"hidden"]
d[u"skills"] = skills
def create_demand(self, access_token, demands: Optional[List[Demand]]):
request_data = {
"RequestInfo": {
"authToken": access_token
},
"Demands": to_json(demands)
}
print("Demand Request-->", request_data)
response = requests.post(urljoin(
config.HOST, "/billing-service/demand/_create?tenantId="),
json=request_data)
res = response.json()
return request_data, res
def extra_info(d, category):
quoted = quote(d[u"handle"])
request = common.make_request(u"/v3.0.0/members/%s/%s/" % (quoted, category))
info = common.to_json(urllib2.urlopen(request).read())[u"result"][u"content"]
del info[u"handle"]
del info[u"userId"]
del info[u"createdBy"]
del info[u"createdAt"]
del info[u"updatedBy"]
del info[u"updatedAt"]
d[category] = info
def wait_until_ready(uri, data_path):
import requests
data = common.to_json(data_path)
start = time.time()
for j in range(iterations):
rsp = None
try:
rsp = call_server.call(uri, data)
except requests.exceptions.ConnectionError as e:
print(f"Calling scoring server: {j}/{iterations}")
if rsp is not None:
print(f"Done waiting for {time.time()-start:5.2f} seconds")
return rsp
time.sleep(sleep_time)
raise Exception(
f"ERROR: Timed out after {iterations} iterations waiting for server to launch"
)
def get_by_id(self, item_id):
"""
:param item_id:
:return: dict - item if found, or None
"""
try:
resp = self.dynamo.get_item(
TableName=self.tablename,
Key={self.partition_key: {
'S': item_id
}})
except ClientError as err:
error_code = err.response['Error']['Code']
logging.exception(f'ERROR {error_code}: {err}')
raise
item = resp.get('Item', None)
if item:
return to_json(item)
def extra_info(d, category):
handle = quote_handle(d[u"handle"])
request = common.make_request(u"/v3/members/%s/%s/" % (handle, category))
raw = common.open_request_and_read(request).decode("utf-8")
info = common.to_json(raw)[u"result"][u"content"]
if info is None:
raise Exception(u"Failed to get `%s` info for %s" % (category, d[u"handle"]))
del info[u"handle"]
del info[u"userId"]
if u"handleLower" in info:
del info[u"handleLower"]
del info[u"createdBy"]
del info[u"createdAt"]
del info[u"updatedBy"]
del info[u"updatedAt"]
d[category] = info
def upload_water_connection(self, access_token):
# print("Water Json : ", self.get_water_json())
request_data = {
"RequestInfo": {
"apiId": "Rainmaker",
"ver": ".01",
"action": "",
"did": "1",
"key": "",
"msgId": "20170310130900|en_IN",
"requesterId": "",
"authToken": access_token
},
"WaterConnection": to_json(self)
}
print("Water connection request payload", request_data)
print(urljoin(config.HOST, "/ws-services/wc/_create?"))
response = requests.post(urljoin(config.HOST, "/ws-services/wc/_create?"), json=request_data)
res = response.json()
return request_data, res
def extra_info(d, category):
handle = quote_handle(d[u"handle"])
request = common.make_request(u"/v3/members/%s/%s/" % (handle, category))
raw = common.open_request_and_read(request).decode("utf-8")
info = common.to_json(raw)[u"result"][u"content"]
if info is None:
raise Exception(u"Failed to get `%s` info for %s" %
(category, d[u"handle"]))
del info[u"handle"]
del info[u"userId"]
if u"handleLower" in info:
del info[u"handleLower"]
del info[u"createdBy"]
del info[u"createdAt"]
del info[u"updatedBy"]
del info[u"updatedAt"]
d[category] = info
def lookup_by_login(self, login_code):
"""
:return: user record, or None
"""
try:
result = self.dynamo.query(
ExpressionAttributeValues={':l': {
'S': login_code
}},
KeyConditionExpression='login_code = :l',
TableName=self.tablename,
IndexName='login_code-index')
items = result['Items']
if len(items) == 0:
return None
elif len(items) > 1:
logging.error(
f'More than one record found for login_code={login_code}')
return None
else:
return to_json(items[0])
except Exception:
raise
def upload_water_collection(self, access_token):
print("hello", access_token)
# print("Water Json : ", self.get_water_json())
request_data = {
"RequestInfo": {
"apiId": "Rainmaker",
"ver": ".01",
"action": "",
"did": "1",
"key": "",
"msgId": "20170310130900|en_IN",
"requesterId": "",
"authToken": access_token
},
"Payment": to_json(self)
}
print(request_data)
print(urljoin(config.HOST, "/payments/_create?"))
time.sleep(0.01)
response = requests.post(urljoin(config.HOST, "/payments/_create?"),
json=request_data)
res = response.json()
return request_data, res
def main():
client = MongoClient()
db = client.topcoder
config = ConfigParser.RawConfigParser()
config.read("config/challenges.ini")
init = config.getboolean("default", "init")
if init:
index = config.getint("default", "page_index")
else:
index = 1
use_proxy = config.getboolean("default", "use_proxy")
common.prepare(use_proxy=use_proxy)
while True:
path = "/v2/challenges/past?type=develop&pageIndex=%d&pageSize=10" % index
raw = common.guarded_read(path)
if '"data": []' in raw:
return
print "Page", index
lists = json.loads(raw)
for challenge in lists["data"]:
cid = challenge["challengeId"]
if filter_out(cid):
continue
if db.challenges.find_one({"challengeId": cid}):
if init:
continue
else:
return
common.random_sleep(1)
print ' ', challenge["challengeName"]
path = "/v2/challenges/" + str(cid)
d = common.to_json(common.guarded_read(path))
path = "/v2/challenges/registrants/" + str(cid)
raw = '{"registrants": %s}' % common.guarded_read(path)
registrants = common.to_json(raw)
path = "/v2/challenges/submissions/" + str(cid)
submissions = common.to_json(common.guarded_read(path))
d.update(registrants)
d.update(submissions)
format_challenge(d)
db.challenges.insert_one(d)
index += 1
if init:
config.set("default", "page_index", index)
with open("config/challenges.ini", "wb") as fp:
config.write(fp)
common.random_sleep(10)
def main():
common.prepare(use_proxy=g_config.use_proxy)
client = MongoClient()
db = client.topcoder
print "Crawling users..."
print "Current:", db.users.count()
if g_config.recrawl_all:
print "Recrawl all users"
if g_config.recheck_invalid_handles:
print "Recheck invalid handles"
invalid = set()
def add_invalid_handle(hdl):
invalid.add(hdl)
with open(INVALID_HANDLES_FPATH, "w") as fp:
for h in sorted(invalid):
try:
fp.write(h.encode("utf-8") + '\n')
except UnicodeDecodeError:
pass
if os.path.exists(INVALID_HANDLES_FPATH):
for line in open(INVALID_HANDLES_FPATH):
line = line.strip()
if line:
invalid.add(line.decode("utf-8"))
handles = set()
query = {u"handle": None}
field = {u"_id": 1}
nb_challeges = db.challenges.count()
for index, challenge in enumerate(db.challenges.find()):
if (index + 1) % 100 == 0:
print "Challenges: %d/%d" % (index + 1, nb_challeges)
for reg in challenge[u"registrants"]:
handle = reg[u"handle"].lower()
for ch in ur" \/":
if ch in handle:
continue
if handle in invalid:
continue
if handle in handles:
continue
if not g_config.recrawl_all:
query[u"handle"] = handle
if db.users.find_one(query, field) is not None:
continue
handles.add(handle)
if g_config.recheck_invalid_handles or g_config.recrawl_all:
handles.update(invalid)
invalid = set()
if os.path.exists(INVALID_HANDLES_FPATH):
os.rename(INVALID_HANDLES_FPATH, INVALID_HANDLES_FPATH + ".bak")
print len(handles), "users to be crawled"
print "-----"
for index, handle in enumerate(handles):
print "[%d/%d]" % (index + 1, len(handles)), handle
while True:
try:
try:
quoted = quote_handle(handle)
except KeyError:
add_invalid_handle(handle)
break
request = common.make_request(u"/v3/members/" + quoted)
s = common.open_request_and_read(request).decode("utf-8")
d = common.to_json(s)[u"result"][u"content"]
try:
refine_user(d)
user_skills(d)
user_stats(d)
user_external_accounts(d)
except:
traceback.print_exc()
add_invalid_handle(handle)
common.random_sleep(DOZE)
break
db.users.insert_one(d)
common.random_sleep(DOZE)
break
except urllib2.HTTPError, e:
if e.code in (
404,
403,
):
add_invalid_handle(handle)
common.random_sleep(DOZE)
break
else:
print "HTTP Error", e.code, e.msg
print e.geturl()
print e.fp.read()
except KeyboardInterrupt:
return
except:
def run(api_uri, data_path):
data = common.to_json(data_path)
return call(api_uri, data)
def main():
config = ConfigParser.RawConfigParser()
config.read("config/users.ini")
use_proxy = config.getboolean("default", "proxy")
common.prepare(use_proxy=use_proxy)
client = MongoClient()
db = client.topcoder
print "Crawling users..."
print "Current:", db.users.count()
invalid = set()
if os.path.exists("config/invalid_handles"):
for line in open("config/invalid_handles"):
line = line.strip()
if line:
invalid.add(line)
handles = set()
for challenge in db.challenges.find():
for reg in challenge["registrants"]:
handle = reg["handle"].lower()
if u' ' in handle or u'/' in handle or u'\\' in handle:
continue
if handle in invalid:
continue
if handle in handles:
continue
if db.users.find_one({u"handle": handle}):
continue
handles.add(handle)
print len(handles), "users to be crawled."
print "-----"
for handle in handles:
print handle
while True:
try:
request = common.make_request(u"/v3.0.0/members/" + quote(handle))
s = urllib2.urlopen(request).read().decode("utf-8")
d = common.to_json(s)[u"result"][u"content"]
refine_user(d)
user_skills(d)
db.users.insert_one(d)
common.random_sleep(1)
break
except urllib2.HTTPError, e:
if e.code == 404 or e.code == 403:
invalid.add(handle)
with open("config/invalid_handles", "w") as fp:
for h in sorted(invalid):
fp.write(h + '\n')
common.random_sleep(1)
break
else:
print "HTTP Error", e.code, e.msg
print e.geturl()
print e.fp.read()
except Exception, e:
print "An unknown exception occurred."
print e
common.random_sleep(20)
def main():
client = MongoClient()
db = client.topcoder
config = ConfigParser.RawConfigParser()
config.read("config/challenges.ini")
init = config.getboolean("default", "init")
if init:
index = config.getint("default", "page_index")
else:
index = 1
use_proxy = config.getboolean("default", "use_proxy")
common.prepare(use_proxy=use_proxy)
while True:
path = "/v2/challenges/past?type=develop&pageIndex=%d&pageSize=10" % index
raw = common.guarded_read(path)
if '"data": []' in raw:
return
print "Page", index
lists = json.loads(raw)
for challenge in lists["data"]:
cid = challenge["challengeId"]
if filter_out(cid):
continue
if db.challenges.find_one({"challengeId": cid}):
if init:
continue
else:
return
common.random_sleep(1)
print ' ', challenge["challengeName"]
path = "/v2/challenges/" + str(cid)
d = common.to_json(common.guarded_read(path))
path = "/v2/challenges/registrants/" + str(cid)
raw = '{"registrants": %s}' % common.guarded_read(path)
registrants = common.to_json(raw)
path = "/v2/challenges/submissions/" + str(cid)
submissions = common.to_json(common.guarded_read(path))
d.update(registrants)
d.update(submissions)
format_challenge(d)
db.challenges.insert_one(d)
index += 1
if init:
config.set("default", "page_index", index)
with open("config/challenges.ini", "wb") as fp:
config.write(fp)
common.random_sleep(10)
def main():
common.prepare(use_proxy=g_config.use_proxy)
client = MongoClient()
db = client.topcoder
print "Crawling users..."
print "Current:", db.users.count()
if g_config.recrawl_all:
print "Recrawl all users"
if g_config.recheck_invalid_handles:
print "Recheck invalid handles"
invalid = set()
def add_invalid_handle(hdl):
invalid.add(hdl)
with open(INVALID_HANDLES_FPATH, "w") as fp:
for h in sorted(invalid):
try:
fp.write(h.encode("utf-8") + '\n')
except UnicodeDecodeError:
pass
if os.path.exists(INVALID_HANDLES_FPATH):
for line in open(INVALID_HANDLES_FPATH):
line = line.strip()
if line:
invalid.add(line.decode("utf-8"))
handles = set()
query = {u"handle": None}
field = {u"_id": 1}
nb_challeges = db.challenges.count()
for index, challenge in enumerate(db.challenges.find()):
if (index + 1) % 100 == 0:
print "Challenges: %d/%d" % (index + 1, nb_challeges)
for reg in challenge[u"registrants"]:
handle = reg[u"handle"].lower()
for ch in ur" \/":
if ch in handle:
continue
if handle in invalid:
continue
if handle in handles:
continue
if not g_config.recrawl_all:
query[u"handle"] = handle
if db.users.find_one(query, field) is not None:
continue
handles.add(handle)
if g_config.recheck_invalid_handles or g_config.recrawl_all:
handles.update(invalid)
invalid = set()
if os.path.exists(INVALID_HANDLES_FPATH):
os.rename(INVALID_HANDLES_FPATH, INVALID_HANDLES_FPATH + ".bak")
print len(handles), "users to be crawled"
print "-----"
for index, handle in enumerate(handles):
print "[%d/%d]" % (index + 1, len(handles)), handle
while True:
try:
try:
quoted = quote_handle(handle)
except KeyError:
add_invalid_handle(handle)
break
request = common.make_request(u"/v3/members/" + quoted)
s = common.open_request_and_read(request).decode("utf-8")
d = common.to_json(s)[u"result"][u"content"]
try:
refine_user(d)
user_skills(d)
user_stats(d)
user_external_accounts(d)
except:
traceback.print_exc()
add_invalid_handle(handle)
common.random_sleep(DOZE)
break
db.users.insert_one(d)
common.random_sleep(DOZE)
break
except urllib2.HTTPError, e:
if e.code in (404, 403,):
add_invalid_handle(handle)
common.random_sleep(DOZE)
break
else:
print "HTTP Error", e.code, e.msg
print e.geturl()
print e.fp.read()
except KeyboardInterrupt:
return
except:
def __repr__(self):
return to_json(self)
def __repr__(self):
return to_json(self.__dict__)
def __repr__(self):
d = self.__dict__.copy()
del d["bytes"]
del d["ts_packets"]
return to_json(d)
