This repository has been archived by the owner on Apr 11, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
/
fetch_user_data.py
executable file
·121 lines (99 loc) · 4.49 KB
/
fetch_user_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
import box, ConfigParser, csv, json, ldap, logging, optparse, os, requests, sys, time
class LdapLookup(object):
def __init__(self, **settings):
ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, False)
self._ldapconn = ldap.initialize(settings["host"])
self._ldapconn.protocol_version = ldap.VERSION3
self._ldapconn.simple_bind_s(settings["user"], settings["passwd"])
self._search_base = settings["search_base"]
def user_info(self, username, attrs=[]):
result = self._ldapconn.search_s(self._search_base, ldap.SCOPE_SUBTREE, 'uid=%s' % username, attrs)
values = {}
for attr in attrs:
try:
values[attr] = ",".join(result[0][1][attr])
except:
values[attr] = None
return values
if __name__ == "__main__":
parser = optparse.OptionParser()
parser.add_option("-f", "--file", dest="filename", help="write data to FILE", metavar="FILE")
parser.add_option("-v", action="count", default=0, dest="verbosity")
(options, args) = parser.parse_args()
logging.basicConfig(filename='/var/log/archive/box/user_list/fetch_user_data.log', filemode='w')
logger = logging.getLogger("box.fetch_user_data")
logger.setLevel(logging.WARN - int(options.verbosity) * 10)
# set logging level of "box.api"
logging.getLogger("box.api").setLevel(logger.getEffectiveLevel())
config = ConfigParser.ConfigParser()
settings_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings.conf")
config.read(settings_file)
if options.filename:
out = csv.writer(open(options.filename, "wb"))
else:
out = csv.writer(sys.stdout)
box = box.BoxApi()
lookup = LdapLookup(**dict(config.items("ldap")))
box_attrs = [f.strip() for f in config.get("user_lookup", "box_attrs").split(",")]
local_attrs = [f.strip() for f in config.get("user_lookup", "ldap_lookup_attrs").split(",")]
#change title of column from "box_tracking_codes" to "box_account_type"
headers = box_attrs[:]
index = headers.index("tracking_codes")
headers[index] = "account_type"
out.writerow(["username", "retrieved_at"] + ["box_%s" % s for s in headers] + ["ldap_%s" % s for s in local_attrs])
box_params = {}
box_params["fields"] = ",".join(box_attrs)
try:
box_params["limit"] = config.get("user_lookup", "box_request_limit")
except:
box_params["limit"] = 100
box_params["offset"] = 0
fetched_count = 0
storage_list = []
while True:
for i in range(6):
if i != 0:
time.sleep(2**i)
try:
#request users
get_box_users = box.request("GET", "/users", params=box_params)
break
except:
#re-initialize box object with current auth token
box = box.BoxApi()
if get_box_users.status_code != requests.codes.ok:
logger.warn("Response code \"%d \"from \"%s\"\n" % (get_box_users.status_code, get_box_users.url))
body = get_box_users.text
box_users = json.loads(body)
if len(box_users["entries"]) < 1:
break
fetched_count += len(box_users["entries"])
logger.info("Fetched %d out of %d." % (fetched_count, box_users["total_count"]))
for box_user in box_users["entries"]:
username = box_user["login"].split("@")[0].lower()
attrs = [username, time.strftime("%Y-%m-%dT%H:%M:%S%z")]
for key in box_attrs:
if key == "tracking_codes":
if box_user[key] != []:
for tracking_code in box_user[key]:
if tracking_code["name"] == "account_type":
attrs.append(tracking_code["value"].lower())
else:
attrs.append("individual")
continue
attrs.append(box_user[key])
#print "\t BOX {key}: {val}".format(key=key, val=user[key])
for i in range(6):
if i != 0:
time.sleep(2**i)
try:
local_user = lookup.user_info(username, local_attrs)
break
except ldap.SERVER_DOWN:
pass
for key in local_attrs:
attrs.append(local_user[key])
#print "\tMCOMM {key}: {val}".format(key=key, val=local_results[key])
out.writerow([unicode(s).encode("ascii", "replace") for s in attrs])
storage_list.append(box_user["space_used"])
box_params["offset"] += box_params["limit"]