def _sanitize_data(self, data, datetime_column, added_by_column):
'''Remove None and empty lists from data. datetime_columnd = added_by or updated_by
Args
ids_to_remove -- data with these IDs will be removed from the dataset
'''
data.update({datetime_column : str(datetime.now())})
data.update({added_by_column : security.retrieve('Username')})
# make all values str and remove if value is None or empty string
data = {k: v for k, v in list(data.items()) if v!=None}
data = dict([a, str(x)] for a, x in data.items() if len(str(x).strip())>0)
return data
def _sanitize_data(self, data, datetime_column, added_by_column):
'''Remove None and empty lists from data. datetime_columnd = added_by or updated_by
Args
ids_to_remove -- data with these IDs will be removed from the dataset
'''
data.update({datetime_column: str(datetime.now())})
data.update({added_by_column: security.retrieve('Username')})
# make all values str and remove if value is None or empty string
data = {k: v for k, v in list(data.items()) if v != None}
data = dict([a, str(x)] for a, x in data.items()
if len(str(x).strip()) > 0)
return data
def __init__(self, server_url, remove_pass_file = True, new_pass_file = True, password_location = '~',log_file = 'molgenis.log', logging_level='ERROR', logfile_mode = 'w'):
'''Initialize Python api to talk to Molgenis Rest API
Args:
server_url (string): The url to the molgenis server (ex: https://molgenis39.target.rug.nl/)
remove_pass_file (bool): If True, remove the files containing the passwords after usage (def: True)
new_pass_file (str): If file with password was not removed after last run, but still want to use a new password this run, set to True. Otherwise uses same password as last run (def: False)
password_location (string): Folder where to put the password files in (def: ~)
log_file (string): Path to write logfile with debug info etc to (def: molgenis.log)
logging_level (string): The level of logging to use. See Python's `logging` manual for info on levels (def: DEBUG)
logfile_mode (string): Mode of writing to logfile, e.g. w for overwrite or a for append, see `logging` manual for more details (def: w)
'''
# because errors in the __init__ function will not go to __exit__, make sure to clean up after error
try:
# if no path is specified in the log_file name, it should be written in the same location where the script is called from,
# not from the location molgenis is located
if not os.sep in log_file:
log_file = os.getcwd()+os.sep+log_file
else:
# if there is a path in log_file, make sure that the folder exists
if not os.path.exists(os.path.dirname(log_file)):
raise OSError('Folder "'+str(os.path.dirname)+'" for writing the molgenis.log file does not exist, change log_file location')
logging.basicConfig(filename = log_file, filemode = logfile_mode)
logging.getLogger().addHandler(logging.StreamHandler())
self.logger = logging.getLogger(__name__)
self.logger.setLevel(level=getattr(logging, logging_level))
self.time_start = timeit.default_timer()
security.overwrite_passphrase_location(password_location)
if new_pass_file:
self.remove_pass_file = True
security.remove_secrets_file()
security.require_username('Username')
security.require_password('Password')
self.server_url = server_url
if not server_url.endswith('api/'):
if not server_url.endswith('/'):
server_url += '/'
server_url += 'api/'
self.session = molgenis.Session(server_url)
self.logger.debug('Trying to log in with data from '+str(security.PASSPHRASE_FILE) +' to: '+server_url+' with username: '******'*'*len(security.retrieve('Username'))+' password: '******'*'*len(security.retrieve('Password')))
try:
self.session.login(security.retrieve('Username'), security.retrieve('Password'))
except requests.exceptions.HTTPError as e:
if 'Not Found for url' in str(e):
self.logger.debug('login failed, trying again')
if server_url.startswith('http:'):
server_url = server_url.replace('http:','https:')
self.session = molgenis.Session(server_url)
elif server_url.startswith('https:'):
server_url = server_url.replace('https:','http:')
self.session = molgenis.Session(server_url)
self.logger.debug('Trying to log in with data from '+str(security.PASSPHRASE_FILE) +' to: '+server_url+' with username: '******'*'*len(security.retrieve('Username'))+' password: '******'*'*len(security.retrieve('Password')))
try:
self.session.login(security.retrieve('Username'), security.retrieve('Password'))
except requests.exceptions.HTTPError as e:
if 'Unauthorized for url' in str(e):
raise requests.exceptions.HTTPError(str(e)+'\nInvalid username or password')
else:
raise
elif 'Unauthorized for url' in str(e):
raise requests.exceptions.HTTPError(str(e)+'\nInvalid username or password')
else:
if len(security.retrieve('Username')) == 0 or len(security.retrieve('Password')) == 0:
raise requests.exceptions.HTTPError(str(e)+'\nError possibly because username or password is empty')
else:
raise
self.entity_meta_data = {}
self.column_meta_data = {}
self.added_rows = 0
self.added_files = 0
self.remove_pass_file = remove_pass_file
except:
self.remove_password_files()
raise
def __init__(self,
server_url,
remove_pass_file=True,
new_pass_file=True,
password_location='~',
log_file='molgenis.log',
logging_level='ERROR',
logfile_mode='w'):
'''Initialize Python api to talk to Molgenis Rest API
Args:
server_url (string): The url to the molgenis server (ex: https://molgenis39.target.rug.nl/)
remove_pass_file (bool): If True, remove the files containing the passwords after usage (def: True)
new_pass_file (str): If file with password was not removed after last run, but still want to use a new password this run, set to True. Otherwise uses same password as last run (def: False)
password_location (string): Folder where to put the password files in (def: ~)
log_file (string): Path to write logfile with debug info etc to (def: molgenis.log)
logging_level (string): The level of logging to use. See Python's `logging` manual for info on levels (def: DEBUG)
logfile_mode (string): Mode of writing to logfile, e.g. w for overwrite or a for append, see `logging` manual for more details (def: w)
'''
# because errors in the __init__ function will not go to __exit__, make sure to clean up after error
try:
# if no path is specified in the log_file name, it should be written in the same location where the script is called from,
# not from the location molgenis is located
if not os.sep in log_file:
log_file = os.getcwd() + os.sep + log_file
else:
# if there is a path in log_file, make sure that the folder exists
if not os.path.exists(os.path.dirname(log_file)):
raise OSError(
'Folder "' + str(os.path.dirname) +
'" for writing the molgenis.log file does not exist, change log_file location'
)
logging.basicConfig(filename=log_file,
filemode=logfile_mode)
logging.getLogger().addHandler(logging.StreamHandler())
self.logger = logging.getLogger(__name__)
self.logger.setLevel(level=getattr(logging, logging_level))
self.time_start = timeit.default_timer()
security.overwrite_passphrase_location(password_location)
if new_pass_file:
self.remove_pass_file = True
security.remove_secrets_file()
security.require_username('Username')
security.require_password('Password')
self.server_url = server_url
if not server_url.endswith('api/'):
if not server_url.endswith('/'):
server_url += '/'
server_url += 'api/'
self.session = molgenis.Session(server_url)
self.logger.debug('Trying to log in with data from ' +
str(security.PASSPHRASE_FILE) + ' to: ' +
server_url + ' with username: '******'*' *
len(security.retrieve('Username')) +
' password: '******'*' * len(security.retrieve('Password')))
try:
self.session.login(security.retrieve('Username'),
security.retrieve('Password'))
except requests.exceptions.HTTPError as e:
if 'Not Found for url' in str(e):
self.logger.debug('login failed, trying again')
if server_url.startswith('http:'):
server_url = server_url.replace(
'http:', 'https:')
self.session = molgenis.Session(server_url)
elif server_url.startswith('https:'):
server_url = server_url.replace(
'https:', 'http:')
self.session = molgenis.Session(server_url)
self.logger.debug(
'Trying to log in with data from ' +
str(security.PASSPHRASE_FILE) + ' to: ' +
server_url + ' with username: '******'*' * len(security.retrieve('Username')) +
' password: '******'*' * len(security.retrieve('Password')))
try:
self.session.login(
security.retrieve('Username'),
security.retrieve('Password'))
except requests.exceptions.HTTPError as e:
if 'Unauthorized for url' in str(e):
raise requests.exceptions.HTTPError(
str(e) +
'\nInvalid username or password')
else:
raise
elif 'Unauthorized for url' in str(e):
raise requests.exceptions.HTTPError(
str(e) + '\nInvalid username or password')
else:
if len(security.retrieve('Username')) == 0 or len(
security.retrieve('Password')) == 0:
raise requests.exceptions.HTTPError(
str(e) +
'\nError possibly because username or password is empty'
)
else:
raise
self.entity_meta_data = {}
self.column_meta_data = {}
self.added_rows = 0
self.added_files = 0
self.remove_pass_file = remove_pass_file
except:
self.remove_password_files()
raise