def _parse_date_time(cls, time):
if isinstance(time, basestring):
if cls._can_be_cast_to_int(time):
return int(time)
try:
time = datetime.datetime.strptime(time, cls._DATE_TIME_FORMAT)
except ValueError:
try:
time = datetime.datetime.strptime(time, cls._DATE_FORMAT)
except ValueError:
raise GenestackException(
'Unexpected datetime string format: %s, '
'specify date in on of the next format: "%s", "%s"' %
(time, cls._DATE_TIME_FORMAT, cls._DATE_FORMAT))
if isinstance(time, datetime.datetime):
diff = time - datetime.datetime(1970, 1, 1)
milliseconds = (diff.days * 24 * 60 * 60 +
diff.seconds) * 1000 + diff.microseconds / 1000
elif isinstance(time, datetime.date):
diff = time - datetime.date(1970, 1, 1)
milliseconds = diff.days * 24 * 60 * 60 * 1000
elif isinstance(time, float):
milliseconds = int(time * 1000)
else:
raise GenestackException('Unexpected datetime input type: %s' %
type(time))
return milliseconds
def add_user(self, user, save=True):
if not user.alias:
raise GenestackException("Cant add user without alias to config")
if user.alias in self.__users:
raise GenestackException("User alias %s is already present" % user.alias)
self.__users[user.alias] = user
if len(self.__users) == 1:
self.set_default_user(user, save=False)
if save:
self.save()
def add_user(self, user, save=True):
if not user.alias:
raise GenestackException('Cannot add user without alias')
if user.alias in self.__users:
raise GenestackException('User alias "%s" already exists' %
user.alias)
self.__users[user.alias] = user
if len(self.__users) == 1:
self.set_default_user(user, save=False)
if save:
self.save()
def get_folder(self, parent, *names, **kwargs):
"""
Find a subfolder (by name) in a folder passed as an accession,
returning accession of that subfolder. If several names are provided,
treat them as a path components for the sub-sub-...-folder down the
folder hierarchy, returning accession of that deepmost folder:
- ``fu.get_folder('GS777', 'RNASeq')`` looks for subfolder with *name*
"RNASeq" in folder with *accession* "GS777", and returns accession of
that "RNASeq" subfolder;
- ``fu.get_folder('GS777', 'Experiments', 'RNASeq')`` looks for
subfolder with *name* "Experiments" in a folder with *accession*
"GS777", then looks for "RNASeq" in "Experiments", and returns the
accession of "RNASeq".
If ``create=True`` is passed as a *kwarg*, all the folders in ``names``
hierarchy will be created (otherwise ``GenestackException`` is raised).
:param parent: accession of folder to search in
:type parent: str
:param \*names: tuple of "path components", a hierarchy of folders to
find
:type names: tuple
:param create: whether to create folders from ``names`` if they don't
exist or not; default is ``False`` (raise
``GenestackException`` if any folder doesn't exist)
:type create: bool
:return: accession of found (or created) subfolder
:rtype: str
:raises GenestackException:
if no name is passed, or folder with required name is not found
(and shouldn't be created)
"""
if not names:
raise GenestackException("At least one path should be specified")
create = bool(kwargs.get('create'))
for path in names:
if create:
parent = self.find_or_create_folder(path, parent=parent)
else:
_parent_accession = self.find_file_by_name(
path, parent=parent, file_class=self.FOLDER)
if _parent_accession is None:
raise GenestackException(
'Cannot find folder with name "%s" '
'in folder with accession: %s' % (path, parent))
parent = _parent_accession
return parent
def __init__(self, connection, application_id=None):
if application_id and self.APPLICATION_ID:
raise GenestackException(
"Application ID specified both as argument and as class variable"
)
self.application_id = application_id or self.APPLICATION_ID
if not self.application_id:
raise GenestackException('Application ID was not specified')
self.connection = connection
# validation of application ID
if len(self.application_id.split('/')) != 2:
raise GenestackException(
'Invalid application ID, expect "{vendor}/{application}" got: %s'
% self.application_id)
def create_file(self,
source_files,
name=None,
params=None,
calculate_checksums=False,
expected_checksums=None,
initialize=False):
"""
Same as the parent method except that intersect applications also need a separate source
file to intersect with, so it treats the last element of the ``source_files`` array as that
file.
"""
if len(source_files) < 2:
raise GenestackException(
'Intersect application requires at least two source files')
main_source_files = source_files[:-1]
other_source_file = source_files[-1]
created_file_accession = super(IntersectApplication, self).create_file(
main_source_files, name, params, calculate_checksums,
expected_checksums, initialize)
self.replace_file_reference(accession=created_file_accession,
key=self.INTERSECT_OTHER_SOURCE_KEY,
accession_to_remove=None,
accession_to_add=other_source_file)
return created_file_accession
def create_dataflow(self, accession, name=None):
"""
Creates a data flow based on the file provenance of the specified file.
The nodes of the data flow can be accessed by the accession of the corresponding files in the file provenance.
:param accession: file accession
:type accession: str
:param name: data flow name
:type name: str
:return: accession of the created data flow file
:rtype: str
:raise GenestackException:
"""
response = self.invoke('initializeApplicationState',
'createFromSources', accession)
if response['type'] == 'newPage':
accession = response['fileInfo']['accession']
elif response['type'] == 'existingPages':
# If file already exists we expect to get the last created file.
# Existing page contains files from first to last (or MAX QUERY)
# TODO: in case there are more files then MAX QUERY (100 ATM),
# the last file in response will not be really last
# (it is almost impossible use case, though)
file_info = response['fileInfos'][-1]
accession = file_info['accession']
else:
raise GenestackException("Unknown response type: %s" %
response['type'])
if name:
FilesUtil(self.connection).replace_metainfo_string_value(
[accession], Metainfo.NAME, name)
return accession
def ask_confirmation(question, default=None):
"""
Ask confirmation and return response as boolean value.
Will be looping until user provides correct credentials.
Does not check if program is attached to tty device.
:param question: question to ask, without [y/n] suffix and question mark.
:param default: default value for empty string. Can be ``'y'``, ``'n'``, and ``None``
:return: whether user confirms
:rtype: bool
"""
if not isatty():
raise GenestackException("Prompt cannot be called")
assert default in ('y', 'n', None), 'Wrong default value, expect "n", "y" or None'
question_suffix = '[%s/%s]' % tuple(x.upper() if x == default else x for x in 'yn')
while True:
text = input('%s %s? ' % (question, question_suffix)).strip().lower()
if not text and default:
text = default
if text in ('y', 'yes'):
return True
if text in ('n', 'no'):
return False
print('Unexpected response please input "y[es]" or "n[o]"')
def get_folder(self, parent, *names, **kwargs):
"""
Finds path recursively. As first argument it accepts any existing folder accession.
For each path in path corresponding folder founded. If folder is not found exception raised,
except key ``create=True`` specified. In that case all folders will be created.
:param parent: parent accession
:type parent: str
:param names: tuple of folder names that should be found/created
:type names: tuple
:param created: set True if missed folder should be created. Default is False
:type created: bool
:return: accession of last folder in paths.
:rtype: str
:raises: GenestackException: when paths are not specified or parent cannot be found.
"""
if not names:
raise GenestackException("At least one path should be specified")
create = bool(kwargs.get('create'))
for path in names:
if create:
parent = self.find_or_create_folder(path, parent=parent)
else:
_parent_accession = self.find_file_by_name(
path, parent=parent, file_class=self.FOLDER)
if _parent_accession is None:
raise Exception(
'Cant find folder with name "%s" in folder with accession: %s'
% (path, parent))
parent = _parent_accession
return parent
def set_default_user(self, user, save=True):
if not user.alias in self.__users:
raise GenestackException('User %s is not present in config users' % user.alias)
if not self.default_user or user.alias != self.default_user.alias:
self.__default_user = user
if save:
self.save()
def check_version(self, version):
"""
Check the version of the client library required by the server.
The server will return a message specifying the latest version and the earliest compatible version.
If the current version is not supported, an exception is raised.
:param version: version in format suitable for distutils.version.StrictVersion
:return: a user-friendly message.
"""
version_map = self.application('genestack/clientVersion').invoke(
'getCurrentVersion')
LATEST = 'latest'
COMPATIBLE = 'compatible'
latest_version = StrictVersion(version_map[LATEST])
my_verison = StrictVersion(version)
if latest_version <= my_verison:
return ''
compatible = StrictVersion(version_map[COMPATIBLE])
update_message = (
'You can update it with the following command:\n'
' pip install git+https://github.com/genestack/python-client@stable\n'
)
if my_verison >= compatible:
return 'Newer version "%s" available, please update.\n%s' % (
latest_version, update_message)
else:
raise GenestackException(
'Your version "%s" is too old, please update to %s.\n%s' %
(my_verison, latest_version, update_message))
def set_default_user(self, user, save=True):
if user.alias not in self.__users:
raise GenestackException('User "%s" does not exist' % user.alias)
if not self.default_user or user.alias != self.default_user.alias:
self.__default_user = user
if save:
self.save()
def load(self):
config_path = self.get_settings_file()
if not os.path.exists(config_path):
return
def get_text(parent, tag):
elements = parent.getElementsByTagName(tag)
if len(elements) == 1:
return elements[0].childNodes[0].data.strip()
return None
with open(config_path) as f:
dom = parse(f)
users = dom.getElementsByTagName('user')
for user in users:
alias = get_text(user, 'alias')
host = get_text(user, 'host')
email = get_text(user, 'email')
password = get_text(user, 'password')
token = get_text(user, 'token')
if not password:
try:
import keyring
password = keyring.get_password(
_PASSWORD_KEYRING, alias)
except Exception as e:
sys.stderr.write(
'Fail to load password for alias "%s": %s\n' %
(alias, e))
if not token:
try:
import keyring
token = keyring.get_password(_TOKEN_KEYRING, alias)
except Exception as e:
sys.stderr.write(
'Fail to load token for alias "%s": %s\n' %
(alias, e))
self.add_user(User(email,
alias=alias,
host=host,
password=password,
token=token),
save=False)
default_user_alias = get_text(dom, 'default_user')
store_raw_text = get_text(dom, 'store_raw')
if store_raw_text == 'True':
self.store_raw = True
elif store_raw_text == 'False':
self.store_raw = False
try:
default_user = self.__users[default_user_alias]
self.set_default_user(default_user, save=False)
except KeyError:
raise GenestackException(
'Cannot set find user: "******" is not present in config users' %
default_user_alias)
def add_date_time(self, key, time):
"""
Add a date.
The time parameter can be passed in one of the following formats:
- :py:class:`datetime.datetime`
- :py:class:`datetime.date`
- :py:class:`str` in format: ``'%Y-%m-%d %H:%M:%S'`` or ``'%Y-%m-%d'``
- number of seconds since the epoch as a floating point number
:param key: key
:type key: str
:param value: time value
:rtype: None
"""
date_time_format = '%Y-%m-%d %H:%M:%S'
date_format = '%Y-%m-%d'
result = Metainfo._create_dict_with_type('datetime')
if isinstance(time, basestring):
try:
time = datetime.datetime.strptime(time, date_time_format)
except ValueError:
try:
time = datetime.datetime.strptime(time, date_format)
except ValueError:
raise GenestackException(
'Unexpected datetime string format: %s, '
'specify date in on of the next format: "%s", "%s"' %
(time, date_time_format, date_format))
if isinstance(time, datetime.datetime):
diff = time - datetime.datetime(1970, 1, 1)
milliseconds = (diff.days * 24 * 60 * 60 +
diff.seconds) * 1000 + diff.microseconds / 1000
elif isinstance(time, datetime.date):
diff = time - datetime.date(1970, 1, 1)
milliseconds = diff.days * 24 * 60 * 60 * 1000
elif isinstance(time, float):
milliseconds = int(time * 1000)
else:
raise GenestackException('Unexpected datetime input type: %s' %
type(time))
result['date'] = xstr(milliseconds)
self.setdefault(key, []).append(result)
def create_experiment(self,
parent=None,
name=None,
description=None,
metainfo=None):
raise GenestackException(
'"create_experiment" is not available anymore, '
'use "FilesUtil.create_dataset" method')
def __init__(self, group, permission):
super(PermissionFileFilter, self).__init__()
if not validate_constant(Permissions, permission):
raise GenestackException("Invalid permission")
self._dict.update(
{'permission': {
'group': group,
'value': permission
}})
def _fill_dataset_metainfo(dataset_metainfo, name):
dataset_metainfo = dataset_metainfo or Metainfo()
if Metainfo.NAME in dataset_metainfo:
raise GenestackException(
'Provided metainfo must not have "%s" field set' %
Metainfo.NAME)
dataset_metainfo.add_string(Metainfo.NAME, name)
return dataset_metainfo
def run(self):
jar_files = [
resolve_jar_file(f) for f in match_jar_globs(self.args.files)
]
if not jar_files:
raise GenestackException('No JAR file was found')
return upload_file(self.connection.application(APPLICATION_ID),
jar_files, self.args.version, self.args.override,
self.args.stable, self.args.scope, self.args.force,
self.args.visibility, self.args.no_wait)
def __are_files_local(links):
if not isinstance(links, list):
links = [links]
if not links:
return False
local_flags = set(map(DataImporter.__is_file_local, links))
if len(local_flags) != 1:
raise GenestackException(
'Different types of links: local and remote')
return local_flags.pop()
def create_sequencing_assay(self,
parent,
name=None,
urls=None,
method=None,
organism=None,
metainfo=None):
raise GenestackException(
'"create_sequencing_assay" is not available anymore, '
'use "create_unaligned_read" method')
def create_microarray_assay(self,
parent,
name=None,
urls=None,
method=None,
organism=None,
metainfo=None):
raise GenestackException(
'"create_microarray_assay" is not available anymore, '
'use "create_microarray_data" method')
def __get_local_path(filepath):
if os.path.exists(filepath):
return True
url_parse = urlparse(filepath)
# TODO check if next check is have any sense.
if url_parse.scheme == '' or url_parse.scheme == 'file':
if not os.path.exists(url_parse.path):
raise GenestackException('Local file is not found: ' +
url_parse.path)
return url_parse.path
return None
def run(self):
app_ids = self.args.app_id_list
if app_ids == ['ALL']:
app_ids = None
elif not all(map(validate_application_id, app_ids)):
return 1
application = self.connection.application(APPLICATION_ID)
version = self.args.version
if not self.args.force and \
not prompt_removing_stable_version(application, app_ids, version):
raise GenestackException('Removing was aborted by user')
return remove_applications(self.connection.application(APPLICATION_ID),
self.args.version, app_ids)
def resolve_jar_file(file_path):
if not os.path.exists(file_path):
raise GenestackException("No such file or directory: %s" % file_path)
if not os.path.isdir(file_path):
return file_path
jar_files = []
for dirpath, dirnames, filenames in os.walk(file_path, followlinks=True):
for f in filenames:
if f.lower().endswith('.jar'):
jar_files.append(os.path.join(dirpath, f))
if len(jar_files) > 1:
raise GenestackException(
'More than one JAR file was found inside %s:\n'
' %s' % (file_path, '\n '.join(jar_files)))
elif not jar_files:
raise GenestackException(
'No JAR files were found within given files/directories: "%s"' %
file_path)
return jar_files[0]
def add_value(self, key, value):
"""
Add a scalar value to a metainfo key.
If adding to an existing key, the value will be appended to the list of existing values.
:param key: key
:type key: str
:param value: value
:type value: MetainfoScalarValue
:rtype None:
"""
if not isinstance(value, MetainfoScalarValue):
raise GenestackException("Value is not an instance of `MetainfoScalarValue`")
self.setdefault(key, []).append(value)
def upload(self):
def do_stuff():
"""
The daemon will look for uploads.
The daemon will quit if one of the following conditions is met:
- all chunks have been processed
- someone set self.finished to True
- the server said that the file upload was complete
- a permanent error was raised (4xx, 5xx)
- the number of RETRY_ATTEMPTS was exceeded for a single chunk
"""
with self.condition:
self.thread_counter += 1
try:
while not self.finished: # daemon working cycle
try:
with self.__iterator_lock:
chunk = next(self.iterator)
except StopIteration:
return
self.__process_chunk(chunk)
except Exception as e:
self.error = str(e)
finally:
with self.condition:
self.thread_counter -= 1
self.condition.notify()
threads = [
Thread(target=do_stuff)
for _ in range(min(NUM_THREADS, self.chunk_count))
]
[thread.setDaemon(True) for thread in threads]
[thread.start() for thread in threads]
with self.condition:
while True:
try:
self.condition.wait()
except (KeyboardInterrupt, SystemExit):
self.error = 'Interrupted by user'
self.finished = True
break
if not self.thread_counter:
break
if self.has_application_result:
return self.application_result
else:
raise GenestackException(
'Fail to upload %s: %s' %
(self.path, self.error or 'unknown error'))
def __add_to_metainfo(metainfo, key, value, value_type, required=False):
"""
Add ``key``: ``value`` pair to ``metainfo``.
Fails if:
* ``metainfo`` already contains ``key``;
* ``key`` is ``required``, but no value is provided by user,
and neither ``metainfo`` has already existing ``key`` (with some value).
:param metainfo: metainfo object to be updated
:type metainfo: Metainfo
:param key: key to add
:type key: str
:param value: single argument to pass to the value class constructor, or list if need to add many values.
:type value: object
:param value_type: value class
:type value_type: T => genestack_client.metainfo_scalar_values.MetainfoScalarValue
:param required: flag if value is required
:type required: bool
:return: None
"""
current_value = metainfo.get(key)
if current_value is not None and value is not None:
raise GenestackException(
'Key "%s" is passed both as function argument '
'and inside metainfo object' % key)
if current_value:
return
if required and value is None:
raise GenestackException(
'Missing required key "%s", '
'it should be passed as function argument '
'or in metainfo object' % key)
if value is not None:
value_list = value if isinstance(value, list) else [value]
for val in value_list:
metainfo.add_value(key, value_type(val))
def find_files(self,
file_filter,
sort_order=SortOrder.DEFAULT,
ascending=False,
offset=0,
limit=MAX_FILE_SEARCH_LIMIT):
"""
Search for files with ``file_filter`` and return dictionary with two key/value pairs:
- ``'total'``: total number (``int``) of files matching the query
- ``'result'``: list of file info dictionaries for subset of matching files
(from ``offset`` to ``offset+limit``). See the documentation of
:py:meth:`~genestack_client.FilesUtil.get_infos` for the structure
of these objects.
:param file_filter: file filter
:type file_filter: FileFilter
:param sort_order: sorting order for the results,
see :py:class:`~genestack_client.files_util.SortOrder`
:type sort_order: str
:param ascending: should the results be in ascending order? (default: False)
:type ascending: bool
:param offset: search offset (default: 0, cannot be negative)
:type offset: int
:param limit: maximum number of results to return (max and default: 100)
:type limit: int
:return: a dictionary with search response
:rtype: dict[str, int|list[dict[str, str|dict]]]
"""
limit = min(self.MAX_FILE_SEARCH_LIMIT, limit)
if offset < 0 or limit < 0:
raise GenestackException("Search offset/limit cannot be negative")
if not validate_constant(SortOrder, sort_order):
raise GenestackException("Invalid sort order")
return self.invoke('findFiles', file_filter.get_dict(), sort_order,
ascending, offset, limit)
def _create_file(
self,
groups,
organism=None,
normalized_input=None,
options=None,
# TODO: remove normalised_input argument.
# This argument is deprecated, use normalized_input instead.
# We perform renaming in a scope of global 's' -> 'z' refactoring (american style english).
normalised_input=None):
# TODO: reomve this after removing the normalised_input argument
if normalized_input and normalised_input:
raise GenestackException(
'Both normalized_input and normalised_input are not allowed')
normalized_input = normalized_input or normalised_input
assignments = [(group_id, accession)
for group_id, group in enumerate(groups, 1)
for accession in group['accessions']]
organism = organism or "unknown organism"
group_names = [
group.get("name") or "Group %d" % i
for i, group in enumerate(groups, 1)
]
group_names = ["Ungrouped"] + group_names
group_descriptions = [
group.get("description") or "No description provided"
for group in groups
]
group_descriptions = ["Ungrouped"] + group_descriptions
options = options or {}
params = {
'accessionList': [acc for group_id, acc in assignments],
'groupIdList': map(str,
[group_id for group_id, acc in assignments]),
'organism': organism,
'groupsNameList': group_names,
'groupsDescriptionList': group_descriptions,
'programOptions': options,
}
if normalized_input:
params['sourcesAccessions'] = normalized_input
return self.invoke('createFile', params)
def __interactive_login(self, connection):
if not isatty():
raise GenestackException("Interactive login is not possible")
connection.check_version()
email = self.email
message = 'Connecting to %s' % self.host
login_by_token = 'by token'
login_by_email = 'by email and password'
login_anonymously = 'anonymously'
choice = interactive_select(
[login_by_token, login_by_email, login_anonymously],
'How do you want to login')
if choice == login_anonymously:
return
while True:
if message:
print(message)
if choice == login_by_email:
input_message = 'e-mail [%s]: ' % email if email and '@' in email else 'e-mail: '
email = raw_input(input_message).strip() or email
password = getpass('password for %s: ' % email)
try:
connection.login(email, password)
self.email = email
self.password = password
return
except GenestackAuthenticationException:
message = (
'Your username and password have been rejected by %s, '
'please try again' % self.host)
else:
token = getpass('token: ')
try:
connection.login_by_token(token)
self.token = token
return
except GenestackAuthenticationException:
message = 'Your token has been rejected by %s, please try again' % self.host
