def _sanitize_data(self, data, datetime_column, added_by_column): '''Remove None and empty lists from data. datetime_columnd = added_by or updated_by Args ids_to_remove -- data with these IDs will be removed from the dataset ''' data.update({datetime_column : str(datetime.now())}) data.update({added_by_column : security.retrieve('Username')}) # make all values str and remove if value is None or empty string data = {k: v for k, v in list(data.items()) if v!=None} data = dict([a, str(x)] for a, x in data.items() if len(str(x).strip())>0) return data
def _sanitize_data(self, data, datetime_column, added_by_column): '''Remove None and empty lists from data. datetime_columnd = added_by or updated_by Args ids_to_remove -- data with these IDs will be removed from the dataset ''' data.update({datetime_column: str(datetime.now())}) data.update({added_by_column: security.retrieve('Username')}) # make all values str and remove if value is None or empty string data = {k: v for k, v in list(data.items()) if v != None} data = dict([a, str(x)] for a, x in data.items() if len(str(x).strip()) > 0) return data
def __init__(self, server_url, remove_pass_file = True, new_pass_file = True, password_location = '~',log_file = 'molgenis.log', logging_level='ERROR', logfile_mode = 'w'): '''Initialize Python api to talk to Molgenis Rest API Args: server_url (string): The url to the molgenis server (ex: https://molgenis39.target.rug.nl/) remove_pass_file (bool): If True, remove the files containing the passwords after usage (def: True) new_pass_file (str): If file with password was not removed after last run, but still want to use a new password this run, set to True. Otherwise uses same password as last run (def: False) password_location (string): Folder where to put the password files in (def: ~) log_file (string): Path to write logfile with debug info etc to (def: molgenis.log) logging_level (string): The level of logging to use. See Python's `logging` manual for info on levels (def: DEBUG) logfile_mode (string): Mode of writing to logfile, e.g. w for overwrite or a for append, see `logging` manual for more details (def: w) ''' # because errors in the __init__ function will not go to __exit__, make sure to clean up after error try: # if no path is specified in the log_file name, it should be written in the same location where the script is called from, # not from the location molgenis is located if not os.sep in log_file: log_file = os.getcwd()+os.sep+log_file else: # if there is a path in log_file, make sure that the folder exists if not os.path.exists(os.path.dirname(log_file)): raise OSError('Folder "'+str(os.path.dirname)+'" for writing the molgenis.log file does not exist, change log_file location') logging.basicConfig(filename = log_file, filemode = logfile_mode) logging.getLogger().addHandler(logging.StreamHandler()) self.logger = logging.getLogger(__name__) self.logger.setLevel(level=getattr(logging, logging_level)) self.time_start = timeit.default_timer() security.overwrite_passphrase_location(password_location) if new_pass_file: self.remove_pass_file = True security.remove_secrets_file() security.require_username('Username') security.require_password('Password') self.server_url = server_url if not server_url.endswith('api/'): if not server_url.endswith('/'): server_url += '/' server_url += 'api/' self.session = molgenis.Session(server_url) self.logger.debug('Trying to log in with data from '+str(security.PASSPHRASE_FILE) +' to: '+server_url+' with username: '******'*'*len(security.retrieve('Username'))+' password: '******'*'*len(security.retrieve('Password'))) try: self.session.login(security.retrieve('Username'), security.retrieve('Password')) except requests.exceptions.HTTPError as e: if 'Not Found for url' in str(e): self.logger.debug('login failed, trying again') if server_url.startswith('http:'): server_url = server_url.replace('http:','https:') self.session = molgenis.Session(server_url) elif server_url.startswith('https:'): server_url = server_url.replace('https:','http:') self.session = molgenis.Session(server_url) self.logger.debug('Trying to log in with data from '+str(security.PASSPHRASE_FILE) +' to: '+server_url+' with username: '******'*'*len(security.retrieve('Username'))+' password: '******'*'*len(security.retrieve('Password'))) try: self.session.login(security.retrieve('Username'), security.retrieve('Password')) except requests.exceptions.HTTPError as e: if 'Unauthorized for url' in str(e): raise requests.exceptions.HTTPError(str(e)+'\nInvalid username or password') else: raise elif 'Unauthorized for url' in str(e): raise requests.exceptions.HTTPError(str(e)+'\nInvalid username or password') else: if len(security.retrieve('Username')) == 0 or len(security.retrieve('Password')) == 0: raise requests.exceptions.HTTPError(str(e)+'\nError possibly because username or password is empty') else: raise self.entity_meta_data = {} self.column_meta_data = {} self.added_rows = 0 self.added_files = 0 self.remove_pass_file = remove_pass_file except: self.remove_password_files() raise
def __init__(self, server_url, remove_pass_file=True, new_pass_file=True, password_location='~', log_file='molgenis.log', logging_level='ERROR', logfile_mode='w'): '''Initialize Python api to talk to Molgenis Rest API Args: server_url (string): The url to the molgenis server (ex: https://molgenis39.target.rug.nl/) remove_pass_file (bool): If True, remove the files containing the passwords after usage (def: True) new_pass_file (str): If file with password was not removed after last run, but still want to use a new password this run, set to True. Otherwise uses same password as last run (def: False) password_location (string): Folder where to put the password files in (def: ~) log_file (string): Path to write logfile with debug info etc to (def: molgenis.log) logging_level (string): The level of logging to use. See Python's `logging` manual for info on levels (def: DEBUG) logfile_mode (string): Mode of writing to logfile, e.g. w for overwrite or a for append, see `logging` manual for more details (def: w) ''' # because errors in the __init__ function will not go to __exit__, make sure to clean up after error try: # if no path is specified in the log_file name, it should be written in the same location where the script is called from, # not from the location molgenis is located if not os.sep in log_file: log_file = os.getcwd() + os.sep + log_file else: # if there is a path in log_file, make sure that the folder exists if not os.path.exists(os.path.dirname(log_file)): raise OSError( 'Folder "' + str(os.path.dirname) + '" for writing the molgenis.log file does not exist, change log_file location' ) logging.basicConfig(filename=log_file, filemode=logfile_mode) logging.getLogger().addHandler(logging.StreamHandler()) self.logger = logging.getLogger(__name__) self.logger.setLevel(level=getattr(logging, logging_level)) self.time_start = timeit.default_timer() security.overwrite_passphrase_location(password_location) if new_pass_file: self.remove_pass_file = True security.remove_secrets_file() security.require_username('Username') security.require_password('Password') self.server_url = server_url if not server_url.endswith('api/'): if not server_url.endswith('/'): server_url += '/' server_url += 'api/' self.session = molgenis.Session(server_url) self.logger.debug('Trying to log in with data from ' + str(security.PASSPHRASE_FILE) + ' to: ' + server_url + ' with username: '******'*' * len(security.retrieve('Username')) + ' password: '******'*' * len(security.retrieve('Password'))) try: self.session.login(security.retrieve('Username'), security.retrieve('Password')) except requests.exceptions.HTTPError as e: if 'Not Found for url' in str(e): self.logger.debug('login failed, trying again') if server_url.startswith('http:'): server_url = server_url.replace( 'http:', 'https:') self.session = molgenis.Session(server_url) elif server_url.startswith('https:'): server_url = server_url.replace( 'https:', 'http:') self.session = molgenis.Session(server_url) self.logger.debug( 'Trying to log in with data from ' + str(security.PASSPHRASE_FILE) + ' to: ' + server_url + ' with username: '******'*' * len(security.retrieve('Username')) + ' password: '******'*' * len(security.retrieve('Password'))) try: self.session.login( security.retrieve('Username'), security.retrieve('Password')) except requests.exceptions.HTTPError as e: if 'Unauthorized for url' in str(e): raise requests.exceptions.HTTPError( str(e) + '\nInvalid username or password') else: raise elif 'Unauthorized for url' in str(e): raise requests.exceptions.HTTPError( str(e) + '\nInvalid username or password') else: if len(security.retrieve('Username')) == 0 or len( security.retrieve('Password')) == 0: raise requests.exceptions.HTTPError( str(e) + '\nError possibly because username or password is empty' ) else: raise self.entity_meta_data = {} self.column_meta_data = {} self.added_rows = 0 self.added_files = 0 self.remove_pass_file = remove_pass_file except: self.remove_password_files() raise