def _get_mention_id(self, user_id, name, screen_name): mid = self.db.select("mentions", "mid", "WHERE user_id = '" + str(user_id) + "' " + "and name = '" + escape(name) + "' " + "and screen_name = '" + escape(screen_name) + "' LIMIT 1;") if mid: return mid[0]['mid'] else: self.db.insert("mentions", {"user_id": user_id, "name": name, "screen_name": screen_name}) mid = self.db.select("mentions", "mid", "WHERE user_id = '" + str(user_id) + "' " + "and name = '" + escape(name) + "' " + "and screen_name = '" + escape(screen_name) + "' LIMIT 1;") return mid[0]['mid']
def send_email(email, subject, html_message): sender_email = app.config['EMAIL_ADDRESS'] sender_pw = escape(app.config['EMAIL_PASSWORD']) dest_email = email msg = MIMEMultipart('alternative') msg['Subject'] = subject msg['From'] = sender_email msg['To'] = dest_email body = MIMEText(html_message, 'html') msg.attach(body) s = smtplib.SMTP_SSL('smtp.gmail.com') s.login(sender_email, sender_pw) s.sendmail(sender_email, dest_email, msg.as_string()) print("EMAIL SENT\n") s.quit()
) sys.exit(1) #open the GROUPS file and load the LIST of groups from JSON groups_file = open(config['GROUPS_FILE'], 'r') #load entire text file and convert to JSON - dictionary groups = json.loads(groups_file.read()) groups_file.close() #loop through the list of groups and #PUT /groups/{gid} for index, group in (enumerate(groups['array'])): #test for empty group if 'gid' in group: gid = helpers.escape(group['gid']) #build the request api_endpoint = '/acs/api/v1/groups/' + gid url = 'http://' + config['DCOS_IP'] + api_endpoint headers = { 'Content-type': 'application/json', 'Authorization': 'token=' + config['TOKEN'], } data = { 'description': group['description'], } #send the request to PUT the new USER try: request = requests.put(url, data=json.dumps(data), headers=headers) request.raise_for_status() #show progress after request
def post_groups_users(DCOS_IP, groups): """ Get the list of groups_users from the load_path provided as an argument, and post it to a DC/OS cluster available at the DCOS_IP argument. """ config = helpers.get_config(env.CONFIG_FILE) try: #open the GROUPS file and load the LIST of groups from JSON groups_users_file = open(env.GROUPS_USERS_FILE, 'r') helpers.log(log_level='INFO', operation='LOAD', objects=['Groups', 'Users'], indx=0, content='** OK **') except IOError as error: helpers.log(log_level='ERROR', operation='LOAD', objects=['Groups', 'Users'], indx=0, content=request.text) helpers.get_input(message=env.MSG_PRESS_ENTER) return False #load entire text file and convert to JSON - dictionary groups_users = json.loads(groups_users_file.read()) groups_users_file.close() for index, group_user in (enumerate(groups_users['array'])): #PUT /groups/{gid}/users/{uid} gid = helpers.escape(group_user['gid']) #array of users for this group_users for index2, user in (enumerate(group_user['users'])): uid = user['user']['uid'] #build the request api_endpoint = '/acs/api/v1/groups/' + gid + '/users/' + uid url = 'http://' + config['DCOS_IP'] + api_endpoint headers = { 'Content-type': 'application/json', 'Authorization': 'token=' + config['TOKEN'], } #send the request to PUT the new USER try: request = requests.put(url, headers=headers) request.raise_for_status() #show progress after request helpers.log(log_level='INFO', operation='PUT', objects=['Groups: ' + gid, 'Users: ' + uid], indx=index2, content=request.status_code) except requests.exceptions.HTTPError as error: helpers.log(log_level='ERROR', operation='PUT', objects=['Groups: ' + gid, 'Users: ' + uid], indx=index2, content=request.text) helpers.log(log_level='INFO', operation='PUT', objects=['Groups', 'Users'], indx=0, content=env.MSG_DONE) helpers.get_input(message=env.MSG_PRESS_ENTER) return True
model.load_state_dict(torch.load(args.model_file)) model.eval() if (not args.freeze_models) and args.interpolated_model: for member in model.members: member.eval() with open("preds.csv", "w") as f: writer = csv.writer(f) writer.writerow(['id','word']) for i, l in enumerate(open("source_test.txt"),1): ls = [[DE.vocab.stoi[w] for w in l.split(' ')]] # no eos or sos. size 1,n_de x_de = Variable(torch.cuda.LongTensor(ls)) _,wordlist,_ = model.predict2(x_de,beamsz=100,gen_len=3) # wordlist is beamsz,3 longstr = ' '.join(['|'.join([EN.vocab.itos[w] for w in trigram]) for trigram in wordlist]) longstr = escape(longstr) writer.writerow([i,longstr]) # showPlot(plot_losses) # TODO: function not added/checked # # visualize only for AttnNetwork # def visualize(sentence_de,bs,nwords,flname): # attns = (SentLen_EN)x(SentLen_DE), sentence_de = ["German_1",...,"German_(SentLen_DE)"] # ls = [[DE.vocab.stoi[w] for w in sentence_de.split(' ')]] # _,wordlist,attns = model.predict2(Variable(torch.cuda.LongTensor(ls)),beamsz=bs,gen_len=nwords) # fig = plt.figure() # ax = fig.add_subplot(111) # cax = ax.matshow(attns.numpy(), cmap='bone') # fig.colorbar(cax) # # Set up axes # ax.set_xticklabels(sentence_de, rotation=90)
def get_groups_users(DCOS_IP, groups): """ Get the list of users that are members of a group from a DC/OS cluster as a JSON blob. Save the groups_users to the text file in the save_path provided. Return the list of groups and users that belong to them as a dictionary. """ #create a dictionary object that will hold all group-to-user memberships groups_users = {'array': []} for index, group in (enumerate(groups['array'])): #append this group as a dictionary to the list groups_users['array'].append({ 'gid': helpers.escape(group['gid']), 'url': group['url'], 'description': group['description'], 'users': [], #initialize users LIST for this group 'permissions': [] #initialize permissions LIST for this group }) #get users for this group from DC/OS api_endpoint = '/acs/api/v1/groups/' + helpers.escape( group['gid']) + '/users' url = 'http://' + DCOS_IP + api_endpoint config = helpers.get_config(env.CONFIG_FILE) headers = { 'Content-type': 'application/json', 'Authorization': 'token=' + config['TOKEN'], } try: request = requests.get( url, headers=headers, ) request.raise_for_status() helpers.log(log_level='INFO', operation='GET', objects=['Groups: ' + group['gid'], 'Users '], indx=index, content=request.status_code) except requests.exceptions.HTTPError as error: helpers.log(log_level='ERROR', operation='GET', objects=['Groups: ' + group['gid'], 'Users '], indx=index, content=request.text) memberships = request.json() #get memberships from the JSON #Loop through the list of groups and get their associated actions for index2, membership in (enumerate(memberships['array'])): #get each user that is a member of this group and append groups_users['array'][index]['users'].append(membership) #get permissions for this group from DC/OS #GET groups/[gid]/permissions api_endpoint = '/acs/api/v1/groups/' + helpers.escape( group['gid']) + '/permissions' url = 'http://' + config['DCOS_IP'] + api_endpoint config = helpers.get_config(env.CONFIG_FILE) headers = { 'Content-type': 'application/json', 'Authorization': 'token=' + config['TOKEN'], } try: request = requests.get( url, headers=headers, ) request.raise_for_status() helpers.log(log_level='INFO', operation='GET', objects=['Groups: ' + group['gid'], 'Permissions'], indx=index2, content=request.status_code) except requests.exceptions.HTTPError as error: helpers.log(log_level='ERROR', operation='GET', objects=['Groups: ' + group['gid'], 'Permissions'], indx=index2, content=request.text) permissions = request.json() #get memberships from the JSON for index2, permission in (enumerate(memberships['array'])): #get each group membership for this user groups_users['array'][index]['permissions'].append(permission) #write dictionary as a JSON object to file groups_users_json = json.dumps(groups_users) #convert to JSON groups_users_file = open(env.GROUPS_USERS_FILE, 'w') groups_users_file.write(groups_users_json) #write to file in raw JSON groups_users_file.close() #flush helpers.log(log_level='INFO', operation='GET', objects=['Groups', 'Users'], indx=0, content=env.MSG_DONE) return groups_users
def post_acls_permissions(DCOS_IP, acls): """ Get the list of acls_permissions from the load_path in the environment parameters, and post it to a DC/OS cluster available at the DCOS_IP argument. """ #loop through the list of ACL permission rules and create the ACLS in the system #/acls/{rid}/groups/{gid}/{action} #/acls/{rid}/users/{uid}/{action} config = helpers.get_config(env.CONFIG_FILE) try: #open the GROUPS file and load the LIST of groups from JSON acls_permissions_file = open(env.ACLS_PERMISSIONS_FILE, 'r') helpers.log(log_level='INFO', operation='LOAD', objects=['ACLs', 'Permissions'], indx=0, content='** OK **') except IOError as error: helpers.log(log_level='ERROR', operation='LOAD', objects=['ACLs', 'Permissions'], indx=0, content=request.text) helpers.get_input(message=env.MSG_PRESS_ENTER) return False #load entire text file and convert to JSON - dictionary acls_permissions = json.loads(acls_permissions_file.read()) acls_permissions_file.close() for index, acl_permission in (enumerate(acls_permissions['array'])): rid = helpers.escape(acl_permission['rid']) #array of users for this acl_permission if 'users' in acl_permission: for index2, user in (enumerate(acl_permission['users'])): #PUT /acls/{rid}/users/{uid}/{action} uid = user['uid'] #array of actions for this user_acl_permission for index3, action in (enumerate(user['actions'])): name = helpers.escape(action['name']) #build the request api_endpoint = '/acs/api/v1/acls/' + rid + '/users/' + uid + '/' + name url = 'http://' + config['DCOS_IP'] + api_endpoint headers = { 'Content-type': 'application/json', 'Authorization': 'token=' + config['TOKEN'], } #send the request to PUT the new USER try: request = requests.put(url, headers=headers) request.raise_for_status() #show progress after request helpers.log(log_level='INFO', operation='PUT', objects=['ACLs: ' + rid, 'Users: ' + uid], indx=index2, content=request.status_code) except requests.exceptions.HTTPError as error: helpers.log(log_level='ERROR', operation='PUT', objects=['ACLs: ' + rid, 'Users: ' + uid], indx=index2, content=request.text) else: print('**DEBUG: no users in acl_permission {}: {}'.format( index, acl_permission)) #array of groups for this acl_permission if 'groups' in acl_permission: for index2, group in (enumerate(acl_permission['groups'])): #PUT /acls/{rid}/groups/{gid}/{action} gid = helpers.escape(group['gid']) #array of actions for this group_acl_permission for index3, action in (enumerate(group['actions'])): name = helpers.escape(action['name']) #build the request api_endpoint = '/acs/api/v1/acls/' + rid + '/groups/' + gid + '/' + name url = 'http://' + config['DCOS_IP'] + api_endpoint headers = { 'Content-type': 'application/json', 'Authorization': 'token=' + config['TOKEN'], } #send the request to PUT the new USER try: request = requests.put(url, headers=headers) request.raise_for_status() #show progress after request helpers.log(log_level='INFO', operation='PUT', objects=['ACLs: ' + rid, 'Groups: ' + gid], indx=index2, content=request.status_code) except requests.exceptions.HTTPError as error: helpers.log(log_level='ERROR', operation='PUT', objects=['ACLs: ' + rid, 'Groups: ' + gid], indx=index2, content=request.text) else: print('**DEBUG: no groups in acl_permission {}: {}'.format( index, acl_permission)) helpers.log(log_level='INFO', operation='PUT', objects=['ACLs', 'Permissions'], indx=0, content=env.MSG_DONE) helpers.get_input(message=env.MSG_PRESS_ENTER) return True
def get_acls_permissions(DCOS_IP, acls): """ Get the list of Permissions for Users and Groups referenced in an ACL. Save the ACLs_permissions to the text file in the save_path provided. Return the list of permissions for users/groups as a dictionary. """ #create a dictionary object that will hold all group-to-user memberships acls_permissions = {'array': []} #loop through the list of ACLs received and get the permissions # /acls/{rid}/permissions for index, acl in (enumerate(acls['array'])): #append this acl as a dictionary to the list acls_permissions['array'].append({ 'rid': helpers.escape(acl['rid']), 'url': acl['url'], 'description': acl['description'], 'users': [], #initialize users LIST for this acl 'groups': [] #initialize groups LIST for this acl }) #get permissions for this ACL from DC/OS #GET acls/[rid]/permissions api_endpoint = '/acs/api/v1/acls/' + helpers.escape( acl['rid']) + '/permissions' url = 'http://' + DCOS_IP + api_endpoint config = helpers.get_config(env.CONFIG_FILE) headers = { 'Content-type': 'application/json', 'Authorization': 'token=' + config['TOKEN'], } try: request = requests.get( url, headers=headers, ) request.raise_for_status() helpers.log(log_level='INFO', operation='GET', objects=['ACLs: ' + acl['rid'], 'Permissions'], indx=index, content=request.status_code) except requests.exceptions.HTTPError as error: helpers.log(log_level='ERROR', operation='GET', objects=['ACLs:' + acl['rid'], 'Permissions'], indx=index, content=request.text) permissions = request.json() #get memberships from the JSON #Loop through the list of user permissions and get their associated actions for index2, user in (enumerate(permissions['users'])): #get each user that is a member of this acl and append to ['users'] acls_permissions['array'][index]['users'].append(user) #Loop through the list of actions for this user and get the action value for index3, action in (enumerate(user['actions'])): #get action from DC/OS #GET /acls/{rid}/users/{uid}/{action} api_endpoint = '/acs/api/v1/acls/' + helpers.escape( acl['rid'] ) + '/users/' + user['uid'] + '/' + action['name'] url = 'http://' + config['DCOS_IP'] + api_endpoint config = helpers.get_config(env.CONFIG_FILE) headers = { 'Content-type': 'application/json', 'Authorization': 'token=' + config['TOKEN'], } try: request = requests.get( url, headers=headers, ) request.raise_for_status() helpers.log(log_level='INFO', operation='GET', objects=[ 'ACLs: ' + acl['rid'], 'Permissions', 'Users', 'Actions' ], indx=index3, content=request.status_code) except requests.exceptions.HTTPError as error: helpers.log(log_level='ERROR', operation='GET', objects=[ 'ACLs: ' + acl['rid'], 'Permissions', 'Users', 'Actions' ], indx=index3, content=request.text) action_value = request.json() #add the value as another field of the action alongside name and url acls_permissions['array'][index]['users'][index2]['actions'][ index3]['value'] = action_value #Repeat loop with groups to get all groups and actions for index2, group in (enumerate(permissions['groups'])): #get each user that is a member of this acl and append to ['users'] acls_permissions['array'][index]['groups'].append(group) #Loop through the list of actions for this user and get the action value for index3, action in (enumerate(group['actions'])): #get action from DC/OS #GET /acls/{rid}/users/{uid}/{action} api_endpoint = '/acs/api/v1/acls/' + helpers.escape( acl['rid']) + '/groups/' + helpers.escape( group['gid']) + '/' + action['name'] url = 'http://' + config['DCOS_IP'] + api_endpoint config = helpers.get_config(env.CONFIG_FILE) headers = { 'Content-type': 'application/json', 'Authorization': 'token=' + config['TOKEN'], } try: request = requests.get( url, headers=headers, ) request.raise_for_status() helpers.log(log_level='INFO', operation='GET', objects=[ 'ACLs: ' + acl['rid'], 'Permissions', 'Groups', 'Actions' ], indx=index3, content=request.status_code) except requests.exceptions.HTTPError as error: helpers.log(log_level='ERROR', operation='GET', objects=[ 'ACLs: ' + acl['rid'], 'Permissions', 'Groups', 'Actions' ], indx=index3, content=request.text) action_value = request.json() #add the value as another field of the action alongside name and url acls_permissions['array'][index]['groups'][index2]['actions'][ index3]['value'] = action_value #done. #write dictionary as a JSON object to file acls_permissions_json = json.dumps(acls_permissions) #convert to JSON acls_permissions_file = open(env.ACLS_PERMISSIONS_FILE, 'w') acls_permissions_file.write( acls_permissions_json) #write to file in raw JSON acls_permissions_file.close() helpers.log(log_level='INFO', operation='GET', objects=['ACLs', 'Permissions'], indx=0, content=env.MSG_DONE) return acls_permissions
def post_acls(DCOS_IP): """ Get the ACL information from the ACLs file path specified in the env parameters, and post it to a DC/OS cluster available at the DCOS_IP argument. """ config = helpers.get_config(env.CONFIG_FILE) try: #open the ACLS file and load the LIST of acls from JSON acls_file = open(env.ACLS_FILE, 'r') helpers.log(log_level='INFO', operation='LOAD', objects=['ACLs'], indx=0, content='** OK **') except IOError as error: helpers.log(log_level='ERROR', operation='LOAD', objects=['ACLs'], indx=0, content=request.text) return False #load entire text file and convert to JSON - dictionary acls = json.loads(acls_file.read()) acls_file.close() #loop through the list of ACL Rules and #PUT /acls/{rid} for index, acl in (enumerate(acls['array'])): rid = helpers.escape(acl['rid']) #build the request api_endpoint = '/acs/api/v1/acls/' + rid url = 'http://' + config['DCOS_IP'] + api_endpoint headers = { 'Content-type': 'application/json', 'Authorization': 'token=' + config['TOKEN'], } data = { 'description': acl['description'], } #send the request to PUT the new USER try: request = requests.put(url, headers=headers, data=json.dumps(data)) request.raise_for_status() #show progress after request helpers.log(log_level='INFO', operation='PUT', objects=['ACLs: ' + rid], indx=0, content=request.status_code) except requests.exceptions.HTTPError as error: helpers.log(log_level='ERROR', operation='PUT', objects=['ACLs: ' + rid], indx=0, content=request.text) helpers.get_input(message=env.MSG_PRESS_ENTER) return True
sys.stdout.write( '** ERROR: Buffer is empty. Please LOAD or GET ACLs before POSTing them.' ) sys.exit(1) #open the ACLS file and load the LIST of ACLs from JSON acls_file = open(config['ACLS_FILE'], 'r') #load entire text file and convert to JSON - dictionary acls = json.loads(acls_file.read()) acls_file.close() #loop through the list of ACL Rules and create the ACLS in the system #PUT /acls/{rid} for index, acl in (enumerate(acls['array'])): rid = helpers.escape(acl['rid']) #build the request api_endpoint = '/acs/api/v1/acls/' + rid url = 'http://' + config['DCOS_IP'] + api_endpoint headers = { 'Content-type': 'application/json', 'Authorization': 'token=' + config['TOKEN'], } data = { 'description': acl['description'], } #send the request to PUT the new USER try: request = requests.put(url, verify=False, data=json.dumps(data),
#save to GROUPS file groups_file = open(config['GROUPS_FILE'], 'w') groups_file.write( groups) #write to file in same raw JSON as obtained from DC/OS groups_file.close() #change the list of groups loaded from file (or DC/OS) to JSON dictionary groups_json = json.loads(groups) #create a dictionary object that will hold all group-to-user memberships groups_users = {'array': []} for index, group in (enumerate(groups_json['array'])): #append this group as a dictionary to the list groups_users['array'].append({ 'gid': helpers.escape(group['gid']), 'url': group['url'], 'description': group['description'], 'users': [], #initialize users LIST for this group 'permissions': [] #initialize permissions LIST for this group }) #get users for this group from DC/OS #GET groups/[gid]/users api_endpoint = '/acs/api/v1/groups/' + helpers.escape( group['gid']) + '/users' url = 'http://' + config['DCOS_IP'] + api_endpoint try: request = requests.get(url, headers=headers, verify=False) request.raise_for_status() sys.stdout.write(
acls_file.write( acls ) #write to file in same raw JSON as obtained from DC/OS acls_file.close() #change the list of acls loaded from file (or DC/OS) to JSON dictionary acls_json = json.loads( acls ) #create a dictionary object that will hold all group-to-user memberships acls_permissions = { 'array' : [] } #loop through the list of ACLs received and get the permissions # /acls/{rid}/permissions for index, acl in ( enumerate( acls_json['array'] ) ): #append this acl as a dictionary to the list acls_permissions['array'].append( { 'rid' : helpers.escape( acl['rid'] ), 'url' : acl['url'], 'description' : acl['description'], 'users' : [], #initialize users LIST for this acl 'groups': [] #initialize groups LIST for this acl } ) #get permissions for this ACL from DC/OS #GET acls/[rid]/permissions api_endpoint = '/acs/api/v1/acls/'+helpers.escape( acl['rid'] )+'/permissions' url = 'http://'+config['DCOS_IP']+api_endpoint try: request = requests.get( url, headers=headers,
def get_users_groups(DCOS_IP, users): """ Get the list of groups that users are members of from a DC/OS cluster as a JSON blob. Save the users_groups to the text file in the save_path provided. Return the list of groups and users that belong to them as a dictionary. """ #create a dictionary object that will hold all user-to-group memberships users_groups = {'array': []} for index, user in (enumerate(users['array'])): #append this user as a dictionary to the list users_groups['array'].append({ 'uid': helpers.escape(user['uid']), 'url': user['url'], 'description': user['description'], 'is_remote': user['is_remote'], 'is_service': user['is_service'], #'public_key': user['public_key'], #group memberships is a list, with each member being a dictionary 'groups': [], #initialize groups LIST for this user 'permissions': [] #initialize permission LIST for this user }) #get groups for this user from DC/OS api_endpoint = '/acs/api/v1/users/' + helpers.escape( user['uid']) + '/groups' url = 'http://' + DCOS_IP + api_endpoint config = helpers.get_config(env.CONFIG_FILE) headers = { 'Content-type': 'application/json', 'Authorization': 'token=' + config['TOKEN'], } try: request = requests.get( url, headers=headers, ) request.raise_for_status() helpers.log(log_level='INFO', operation='GET', objects=['Users: ' + user['uid'], 'Groups'], indx=index, content=request.status_code) except requests.exceptions.HTTPError as error: helpers.log(log_level='ERROR', operation='GET', objects=['Users:' + user['uid'], 'Groups'], indx=index, content=request.text) memberships = request.json() #get memberships from the JSON for index2, membership in (enumerate(memberships['array'])): #get each group membership for this user users_groups['array'][index]['groups'].append({ 'membershipurl': membership['membershipurl'], 'group': { 'gid': helpers.escape(membership['group']['gid']), 'url': membership['group']['url'], 'description': membership['group']['description'] } }) #TODO #get permissions for this user from DC/OS???? #GET users/[uid]/permissions api_endpoint = '/acs/api/v1/users/' + helpers.escape( user['uid']) + '/permissions' url = 'http://' + config['DCOS_IP'] + api_endpoint config = helpers.get_config(env.CONFIG_FILE) headers = { 'Content-type': 'application/json', 'Authorization': 'token=' + config['TOKEN'], } try: request = requests.get( url, headers=headers, ) request.raise_for_status() helpers.log(log_level='INFO', operation='GET', objects=['Users: ' + user['uid'], 'Permissions'], indx=index2, content=request.status_code) except requests.exceptions.HTTPError as error: helpers.log(log_level='ERROR', operation='GET', objects=['Users: ' + user['uid'], 'Permissions'], indx=index2, content=request.text) permissions = request.json() #get memberships from the JSON for index2, permission in (enumerate(memberships['array'])): #get each group membership for this user users_groups['array'][index]['permissions'].append(permission) #write dictionary as a JSON object to file users_groups_json = json.dumps(users_groups) #convert to JSON users_groups_file = open(env.USERS_GROUPS_FILE, 'w') users_groups_file.write(users_groups_json) #write to file in raw JSON users_groups_file.close() #flush helpers.log(log_level='INFO', operation='GET', objects=['Users', 'Groups'], indx=0, content='* DONE. *') return users_groups