Exemple #1
0
 def lookup_by_name(self, first_name, last_name):
     """
     :return: a tuple of (user_id, is_err)
     If user is found, returns (user_id, False)
     If user is not found, this is not an error [returns (None, False)]
     """
     if not (first_name and last_name):
         return response(
             400, f'Missing name: "{first_name}" "{last_name}"'), True
     try:
         result = self.dynamo.query(
             ExpressionAttributeValues={
                 ':fn': {
                     'S': first_name
                 },
                 ':ln': {
                     'S': last_name
                 }
             },
             KeyConditionExpression='first_name = :fn AND last_name = :ln',
             TableName=self.tablename,
             IndexName='full_name-index')
         items = result['Items']
         if len(items) == 0:
             return (None, False)
         elif len(items) > 1:
             return response(
                 500,
                 f'More than one person named "{first_name}" "{last_name}"'
             ), True
         else:
             item = to_json(items[0])
             return item['user_id'], False
     except Exception:
         raise
Exemple #2
0
def render_project_json(project, tfstate):
    if len(tfstate):
        role = tfstate['apex_function_role']
        if role:
            project['role'] = role
        project = c.deep_merge(project, {"environment": tfstate})
    return c.to_json(project, False)
Exemple #3
0
def render_project_json(project, tfstate):
	if len( tfstate ):
		role = tfstate['apex_function_role']
		if role:
			project['role'] = role
		project = c.deep_merge(project, {"environment": tfstate})
	return c.to_json(project, False)
 def get_all(self):
     """
     :return: a list of all items. At scale, this should be paginated.
     """
     data = self.dynamo.scan(TableName=self.tablename)
     items = [to_json(item) for item in data['Items']]
     return items
def user_skills(d):
    quoted = quote(d[u"handle"])
    request = common.make_request(u"/v3.0.0/members/%s/skills/" % quoted)
    skills = common.to_json(urllib2.urlopen(request).read())
    skills = skills[u"result"][u"content"][u"skills"]

    for dd in skills.values():
        del dd[u"hidden"]

    d[u"skills"] = skills
Exemple #6
0
def user_skills(d):
    handle = quote_handle(d[u"handle"])
    request = common.make_request(u"/v3/members/%s/skills/" % handle)
    raw = common.open_request_and_read(request).decode("utf-8")
    skills = common.to_json(raw)
    skills = skills[u"result"][u"content"][u"skills"]

    for dd in skills.values():
        del dd[u"hidden"]

    d[u"skills"] = skills
Exemple #7
0
def user_skills(d):
    handle = quote_handle(d[u"handle"])
    request = common.make_request(u"/v3/members/%s/skills/" % handle)
    raw = common.open_request_and_read(request).decode("utf-8")
    skills = common.to_json(raw)
    skills = skills[u"result"][u"content"][u"skills"]

    for dd in skills.values():
        del dd[u"hidden"]

    d[u"skills"] = skills
 def create_demand(self, access_token, demands: Optional[List[Demand]]):
     request_data = {
         "RequestInfo": {
             "authToken": access_token
         },
         "Demands": to_json(demands)
     }
     print("Demand Request-->", request_data)
     response = requests.post(urljoin(
         config.HOST, "/billing-service/demand/_create?tenantId="),
                              json=request_data)
     res = response.json()
     return request_data, res
def extra_info(d, category):
    quoted = quote(d[u"handle"])
    request = common.make_request(u"/v3.0.0/members/%s/%s/" % (quoted, category))
    info = common.to_json(urllib2.urlopen(request).read())[u"result"][u"content"]

    del info[u"handle"]
    del info[u"userId"]

    del info[u"createdBy"]
    del info[u"createdAt"]
    del info[u"updatedBy"]
    del info[u"updatedAt"]

    d[category] = info
def wait_until_ready(uri, data_path):
    import requests
    data = common.to_json(data_path)
    start = time.time()
    for j in range(iterations):
        rsp = None
        try:
            rsp = call_server.call(uri, data)
        except requests.exceptions.ConnectionError as e:
            print(f"Calling scoring server: {j}/{iterations}")
        if rsp is not None:
            print(f"Done waiting for {time.time()-start:5.2f} seconds")
            return rsp
        time.sleep(sleep_time)
    raise Exception(
        f"ERROR: Timed out after {iterations} iterations waiting for server to launch"
    )
 def get_by_id(self, item_id):
     """
     :param item_id:
     :return: dict - item if found, or None
     """
     try:
         resp = self.dynamo.get_item(
             TableName=self.tablename,
             Key={self.partition_key: {
                 'S': item_id
             }})
     except ClientError as err:
         error_code = err.response['Error']['Code']
         logging.exception(f'ERROR {error_code}: {err}')
         raise
     item = resp.get('Item', None)
     if item:
         return to_json(item)
Exemple #12
0
def extra_info(d, category):
    handle = quote_handle(d[u"handle"])
    request = common.make_request(u"/v3/members/%s/%s/" % (handle, category))
    raw = common.open_request_and_read(request).decode("utf-8")
    info = common.to_json(raw)[u"result"][u"content"]

    if info is None:
        raise Exception(u"Failed to get `%s` info for %s" % (category, d[u"handle"]))

    del info[u"handle"]
    del info[u"userId"]

    if u"handleLower" in info:
        del info[u"handleLower"]

    del info[u"createdBy"]
    del info[u"createdAt"]
    del info[u"updatedBy"]
    del info[u"updatedAt"]

    d[category] = info
    def upload_water_connection(self, access_token):
        # print("Water Json : ", self.get_water_json())
        request_data = {
            "RequestInfo": {
                "apiId": "Rainmaker",
                "ver": ".01",
                "action": "",
                "did": "1",
                "key": "",
                "msgId": "20170310130900|en_IN",
                "requesterId": "",
                "authToken": access_token
            },
            "WaterConnection":  to_json(self)

        }
        print("Water connection request payload", request_data)
        print(urljoin(config.HOST, "/ws-services/wc/_create?"))
        response = requests.post(urljoin(config.HOST, "/ws-services/wc/_create?"), json=request_data)
        res = response.json()
        return request_data, res
Exemple #14
0
def extra_info(d, category):
    handle = quote_handle(d[u"handle"])
    request = common.make_request(u"/v3/members/%s/%s/" % (handle, category))
    raw = common.open_request_and_read(request).decode("utf-8")
    info = common.to_json(raw)[u"result"][u"content"]

    if info is None:
        raise Exception(u"Failed to get `%s` info for %s" %
                        (category, d[u"handle"]))

    del info[u"handle"]
    del info[u"userId"]

    if u"handleLower" in info:
        del info[u"handleLower"]

    del info[u"createdBy"]
    del info[u"createdAt"]
    del info[u"updatedBy"]
    del info[u"updatedAt"]

    d[category] = info
Exemple #15
0
 def lookup_by_login(self, login_code):
     """
     :return: user record, or None
     """
     try:
         result = self.dynamo.query(
             ExpressionAttributeValues={':l': {
                 'S': login_code
             }},
             KeyConditionExpression='login_code = :l',
             TableName=self.tablename,
             IndexName='login_code-index')
         items = result['Items']
         if len(items) == 0:
             return None
         elif len(items) > 1:
             logging.error(
                 f'More than one record found for login_code={login_code}')
             return None
         else:
             return to_json(items[0])
     except Exception:
         raise
Exemple #16
0
 def upload_water_collection(self, access_token):
     print("hello", access_token)
     # print("Water Json : ", self.get_water_json())
     request_data = {
         "RequestInfo": {
             "apiId": "Rainmaker",
             "ver": ".01",
             "action": "",
             "did": "1",
             "key": "",
             "msgId": "20170310130900|en_IN",
             "requesterId": "",
             "authToken": access_token
         },
         "Payment": to_json(self)
     }
     print(request_data)
     print(urljoin(config.HOST, "/payments/_create?"))
     time.sleep(0.01)
     response = requests.post(urljoin(config.HOST, "/payments/_create?"),
                              json=request_data)
     res = response.json()
     return request_data, res
Exemple #17
0
def main():
    client = MongoClient()
    db = client.topcoder

    config = ConfigParser.RawConfigParser()
    config.read("config/challenges.ini")

    init = config.getboolean("default", "init")

    if init:
        index = config.getint("default", "page_index")
    else:
        index = 1

    use_proxy = config.getboolean("default", "use_proxy")
    common.prepare(use_proxy=use_proxy)

    while True:
        path = "/v2/challenges/past?type=develop&pageIndex=%d&pageSize=10" % index
        raw = common.guarded_read(path)

        if '"data": []' in raw:
            return

        print "Page", index

        lists = json.loads(raw)

        for challenge in lists["data"]:
            cid = challenge["challengeId"]

            if filter_out(cid):
                continue

            if db.challenges.find_one({"challengeId": cid}):
                if init:
                    continue
                else:
                    return

            common.random_sleep(1)

            print ' ', challenge["challengeName"]

            path = "/v2/challenges/" + str(cid)
            d = common.to_json(common.guarded_read(path))

            path = "/v2/challenges/registrants/" + str(cid)
            raw = '{"registrants": %s}' % common.guarded_read(path)
            registrants = common.to_json(raw)

            path = "/v2/challenges/submissions/" + str(cid)
            submissions = common.to_json(common.guarded_read(path))

            d.update(registrants)
            d.update(submissions)
            format_challenge(d)

            db.challenges.insert_one(d)

        index += 1

        if init:
            config.set("default", "page_index", index)
            with open("config/challenges.ini", "wb") as fp:
                config.write(fp)

        common.random_sleep(10)
Exemple #18
0
def main():
    common.prepare(use_proxy=g_config.use_proxy)

    client = MongoClient()
    db = client.topcoder

    print "Crawling users..."
    print "Current:", db.users.count()

    if g_config.recrawl_all:
        print "Recrawl all users"

    if g_config.recheck_invalid_handles:
        print "Recheck invalid handles"

    invalid = set()

    def add_invalid_handle(hdl):
        invalid.add(hdl)

        with open(INVALID_HANDLES_FPATH, "w") as fp:
            for h in sorted(invalid):
                try:
                    fp.write(h.encode("utf-8") + '\n')
                except UnicodeDecodeError:
                    pass

    if os.path.exists(INVALID_HANDLES_FPATH):
        for line in open(INVALID_HANDLES_FPATH):
            line = line.strip()
            if line:
                invalid.add(line.decode("utf-8"))

    handles = set()

    query = {u"handle": None}
    field = {u"_id": 1}

    nb_challeges = db.challenges.count()
    for index, challenge in enumerate(db.challenges.find()):
        if (index + 1) % 100 == 0:
            print "Challenges: %d/%d" % (index + 1, nb_challeges)

        for reg in challenge[u"registrants"]:
            handle = reg[u"handle"].lower()

            for ch in ur" \/":
                if ch in handle:
                    continue

            if handle in invalid:
                continue

            if handle in handles:
                continue

            if not g_config.recrawl_all:
                query[u"handle"] = handle
                if db.users.find_one(query, field) is not None:
                    continue

            handles.add(handle)

    if g_config.recheck_invalid_handles or g_config.recrawl_all:
        handles.update(invalid)
        invalid = set()

        if os.path.exists(INVALID_HANDLES_FPATH):
            os.rename(INVALID_HANDLES_FPATH, INVALID_HANDLES_FPATH + ".bak")

    print len(handles), "users to be crawled"
    print "-----"

    for index, handle in enumerate(handles):
        print "[%d/%d]" % (index + 1, len(handles)), handle

        while True:
            try:
                try:
                    quoted = quote_handle(handle)
                except KeyError:
                    add_invalid_handle(handle)

                    break

                request = common.make_request(u"/v3/members/" + quoted)
                s = common.open_request_and_read(request).decode("utf-8")
                d = common.to_json(s)[u"result"][u"content"]

                try:
                    refine_user(d)
                    user_skills(d)
                    user_stats(d)
                    user_external_accounts(d)
                except:
                    traceback.print_exc()

                    add_invalid_handle(handle)

                    common.random_sleep(DOZE)
                    break

                db.users.insert_one(d)

                common.random_sleep(DOZE)
                break
            except urllib2.HTTPError, e:
                if e.code in (
                        404,
                        403,
                ):
                    add_invalid_handle(handle)

                    common.random_sleep(DOZE)
                    break
                else:
                    print "HTTP Error", e.code, e.msg
                    print e.geturl()
                    print e.fp.read()
            except KeyboardInterrupt:
                return
            except:
def run(api_uri, data_path):
    data = common.to_json(data_path)
    return call(api_uri, data)
def main():
    config = ConfigParser.RawConfigParser()
    config.read("config/users.ini")

    use_proxy = config.getboolean("default", "proxy")
    common.prepare(use_proxy=use_proxy)

    client = MongoClient()
    db = client.topcoder

    print "Crawling users..."
    print "Current:", db.users.count()

    invalid = set()

    if os.path.exists("config/invalid_handles"):
        for line in open("config/invalid_handles"):
            line = line.strip()
            if line:
                invalid.add(line)

    handles = set()

    for challenge in db.challenges.find():
        for reg in challenge["registrants"]:
            handle = reg["handle"].lower()

            if u' ' in handle or u'/' in handle or u'\\' in handle:
                continue

            if handle in invalid:
                continue

            if handle in handles:
                continue

            if db.users.find_one({u"handle": handle}):
                continue

            handles.add(handle)

    print len(handles), "users to be crawled."
    print "-----"

    for handle in handles:
        print handle

        while True:
            try:
                request = common.make_request(u"/v3.0.0/members/" + quote(handle))
                s = urllib2.urlopen(request).read().decode("utf-8")

                d = common.to_json(s)[u"result"][u"content"]
                refine_user(d)

                user_skills(d)

                db.users.insert_one(d)

                common.random_sleep(1)
                break

            except urllib2.HTTPError, e:
                if e.code == 404 or e.code == 403:
                    invalid.add(handle)

                    with open("config/invalid_handles", "w") as fp:
                        for h in sorted(invalid):
                            fp.write(h + '\n')

                    common.random_sleep(1)
                    break
                else:
                    print "HTTP Error", e.code, e.msg
                    print e.geturl()
                    print e.fp.read()
            except Exception, e:
                print "An unknown exception occurred."
                print e

            common.random_sleep(20)
def main():
    client = MongoClient()
    db = client.topcoder

    config = ConfigParser.RawConfigParser()
    config.read("config/challenges.ini")

    init = config.getboolean("default", "init")

    if init:
        index = config.getint("default", "page_index")
    else:
        index = 1

    use_proxy = config.getboolean("default", "use_proxy")
    common.prepare(use_proxy=use_proxy)

    while True:
        path = "/v2/challenges/past?type=develop&pageIndex=%d&pageSize=10" % index
        raw = common.guarded_read(path)

        if '"data": []' in raw:
            return

        print "Page", index

        lists = json.loads(raw)

        for challenge in lists["data"]:
            cid = challenge["challengeId"]

            if filter_out(cid):
                continue

            if db.challenges.find_one({"challengeId": cid}):
                if init:
                    continue
                else:
                    return

            common.random_sleep(1)

            print ' ', challenge["challengeName"]

            path = "/v2/challenges/" + str(cid)
            d = common.to_json(common.guarded_read(path))

            path = "/v2/challenges/registrants/" + str(cid)
            raw = '{"registrants": %s}' % common.guarded_read(path)
            registrants = common.to_json(raw)

            path = "/v2/challenges/submissions/" + str(cid)
            submissions = common.to_json(common.guarded_read(path))

            d.update(registrants)
            d.update(submissions)
            format_challenge(d)

            db.challenges.insert_one(d)

        index += 1

        if init:
            config.set("default", "page_index", index)
            with open("config/challenges.ini", "wb") as fp:
                config.write(fp)

        common.random_sleep(10)
Exemple #22
0
def main():
    common.prepare(use_proxy=g_config.use_proxy)

    client = MongoClient()
    db = client.topcoder

    print "Crawling users..."
    print "Current:", db.users.count()

    if g_config.recrawl_all:
        print "Recrawl all users"

    if g_config.recheck_invalid_handles:
        print "Recheck invalid handles"

    invalid = set()

    def add_invalid_handle(hdl):
        invalid.add(hdl)

        with open(INVALID_HANDLES_FPATH, "w") as fp:
            for h in sorted(invalid):
                try:
                    fp.write(h.encode("utf-8") + '\n')
                except UnicodeDecodeError:
                    pass

    if os.path.exists(INVALID_HANDLES_FPATH):
        for line in open(INVALID_HANDLES_FPATH):
            line = line.strip()
            if line:
                invalid.add(line.decode("utf-8"))

    handles = set()

    query = {u"handle": None}
    field = {u"_id": 1}

    nb_challeges = db.challenges.count()
    for index, challenge in enumerate(db.challenges.find()):
        if (index + 1) % 100 == 0:
            print "Challenges: %d/%d" % (index + 1, nb_challeges)

        for reg in challenge[u"registrants"]:
            handle = reg[u"handle"].lower()

            for ch in ur" \/":
                if ch in handle:
                    continue

            if handle in invalid:
                continue

            if handle in handles:
                continue

            if not g_config.recrawl_all:
                query[u"handle"] = handle
                if db.users.find_one(query, field) is not None:
                    continue

            handles.add(handle)

    if g_config.recheck_invalid_handles or g_config.recrawl_all:
        handles.update(invalid)
        invalid = set()

        if os.path.exists(INVALID_HANDLES_FPATH):
            os.rename(INVALID_HANDLES_FPATH, INVALID_HANDLES_FPATH + ".bak")

    print len(handles), "users to be crawled"
    print "-----"

    for index, handle in enumerate(handles):
        print "[%d/%d]" % (index + 1, len(handles)), handle

        while True:
            try:
                try:
                    quoted = quote_handle(handle)
                except KeyError:
                    add_invalid_handle(handle)

                    break

                request = common.make_request(u"/v3/members/" + quoted)
                s = common.open_request_and_read(request).decode("utf-8")
                d = common.to_json(s)[u"result"][u"content"]

                try:
                    refine_user(d)
                    user_skills(d)
                    user_stats(d)
                    user_external_accounts(d)
                except:
                    traceback.print_exc()

                    add_invalid_handle(handle)

                    common.random_sleep(DOZE)
                    break

                db.users.insert_one(d)

                common.random_sleep(DOZE)
                break
            except urllib2.HTTPError, e:
                if e.code in (404, 403,):
                    add_invalid_handle(handle)

                    common.random_sleep(DOZE)
                    break
                else:
                    print "HTTP Error", e.code, e.msg
                    print e.geturl()
                    print e.fp.read()
            except KeyboardInterrupt:
                return
            except:
Exemple #23
0
 def __repr__(self):
     return to_json(self)
Exemple #24
0
 def __repr__(self):
     return to_json(self.__dict__)
Exemple #25
0
 def __repr__(self):
     d = self.__dict__.copy()
     del d["bytes"]
     del d["ts_packets"]
     return to_json(d)