def get_simplified_property_mapping(index_name, property_id):
"""
:param index_name: index name where the property belongs
:param property_id: id of the property to check
:return: a dict with the mappings of the property in the index given as parameter
"""
app_logging.debug(f'Getting mapping of {property_id} in {index_name}')
index_mapping = get_index_mapping(index_name)
property_accessor = property_id.replace('.', '.properties.')
property_raw_mapping = dict_property_access.get_property_value(index_mapping, property_accessor)
if property_raw_mapping is None:
return None
label, label_mini = utils.get_labels_from_property_name(index_name, property_id)
simplified_mapping = {
'type': get_simplified_property_type(property_raw_mapping),
'aggregatable': get_simplified_property_aggregatability(property_raw_mapping),
'sortable': get_simplified_property_aggregatability(property_raw_mapping),
'label': label,
'label_mini': label_mini
}
return simplified_mapping
def get_es_data(index_name, raw_es_query, raw_context, raw_contextual_sort_data):
"""
:param index_name: name of the index to query
:param raw_es_query: stringifyied version of the query to send to elasticsearch
:param raw_context: stringifyied version of a JSON object describing the context of the query
:param id_property: property that identifies every item. Required when context provided
:param raw_contextual_sort_data: description of sorting if sorting by contextual properties
:return: Returns the json response from elasticsearch and some metadata if necessary
"""
if raw_context is None:
app_logging.debug('No context detected')
es_query = json.loads(raw_es_query)
es_response = es_data.get_es_response(index_name, es_query)
response = {
'es_response': es_response,
}
return response
app_logging.debug(f'Using context: {raw_context}')
es_response, metadata = get_items_with_context(index_name, raw_es_query, raw_context, raw_contextual_sort_data)
response = {
'es_response': es_response,
'metadata': metadata
}
return response
def save_shortened_url(long_url, url_hash):
"""
Saves the shortened url to es
:param long_url: full url to save
:param url_hash: hash of the url
"""
now = datetime.utcnow()
time_delta = timedelta(
days=RUN_CONFIG.get('url_shortening').get('days_valid'))
expiration_date = now + time_delta
expires = expiration_date.timestamp() * 1000
index_name = RUN_CONFIG.get('url_shortening').get('index_name')
document = {
'long_url': long_url,
'hash': url_hash,
'expires': expires,
'creation_date_2': int(now.timestamp() * 1000)
}
dry_run = RUN_CONFIG.get('url_shortening').get('dry_run')
if dry_run:
app_logging.debug(
f'Dry run is true, not saving the document {document} to the index {index_name}'
)
else:
es_data.save_es_doc(index_name, document, refresh='wait_for')
return expiration_date
def get_url_shortening(url_hash):
"""
:param url_hash: hash of the url to look for
:return: url shortening dict from elasticsearch
"""
index_name = RUN_CONFIG.get('url_shortening').get('index_name')
es_query = {
"query": {
"query_string": {
"query": f'"{url_hash}"',
"default_field": "hash"
}
}
}
shortening_response = es_data.get_es_response(index_name,
es_query,
ignore_cache=True)
total_hits = shortening_response['hits']['total']['value']
app_logging.debug(f'total_hits {total_hits}')
if shortening_response['hits']['total']['value'] == 0:
return None
raw_document = shortening_response['hits']['hits'][0]
return raw_document
def get_merged_prop_config(self, index_name, prop_id,
es_property_description,
property_override_description):
"""
:param index_name: name of the index to which the property belongs
:param prop_id: full path of the property, such as '_metadata.assay_data.assay_subcellular_fraction'
:param es_property_description: dict describing the property taken from es
:param property_override_description: dict describing the property taken from the override
:return: the merged configuration between what was found in es and the override config
"""
found_in_es = es_property_description is not None
app_logging.debug(
f'Property {prop_id} of index {index_name} found_in_es: {found_in_es}'
)
found_in_override = property_override_description is not None
app_logging.debug(
f'Property {prop_id} of index {index_name} found_in_override: {found_in_override}'
)
if not found_in_es and not found_in_override:
raise self.PropertiesConfigurationManagerError(
f'The property {prop_id} of index {index_name} does not exist in elasticsearch or as virtual property'
)
is_virtual = not found_in_es and found_in_override
app_logging.debug(
f'Property {prop_id} of index {index_name} is_virtual: {is_virtual}'
)
if not is_virtual:
return {
'index_name': index_name,
'prop_id': prop_id,
**es_property_description,
**(property_override_description if property_override_description is not None else {})
}
base_config = {
'index_name': index_name,
'prop_id': prop_id,
'is_virtual': True
}
based_on = property_override_description.get('based_on')
is_contextual = based_on is None
app_logging.debug(
f'Property {prop_id} of index {index_name} is_contextual: {is_contextual}'
)
if not is_contextual:
app_logging.debug(
f'Property {prop_id} of index {index_name} based_on: {based_on}'
)
return self.get_virtual_non_contextual_property_config(
base_config, based_on, property_override_description)
return self.get_virtual_contextual_property_config(
base_config, property_override_description)
def update_es_doc(index_name, updated_fields, doc_id):
"""
Updates the document with the document provided
:param doc_id: id of the document to update
:param index_name: index where to save the document
:param updated_fields: updated fields of the document to save
"""
app_logging.debug(f'Updating the document with id {doc_id} with this {updated_fields} on the index {index_name}')
result = ES.update(index=index_name, body=updated_fields, doc_type='_doc', id=doc_id)
app_logging.debug(f'Result {result}')
def save_es_doc(index_name, document, **kwargs):
"""
Saves to elasticsearch the document at the index indicated by parameter
:param index_name: index in which to save the document
:param document: document to save
:param kwargs: keyword args to pass to the index function:
https://elasticsearch-py.readthedocs.io/en/master/api.html#elasticsearch.Elasticsearch.index
"""
app_logging.debug(f'Saving the document {document} to the index {index_name}')
result = ES.index(index=index_name, body=document, doc_type='_doc', **kwargs)
app_logging.debug(f'Result {result}')
def parse_args_and_submit_job(job_type, form_args, file_args):
app_logging.debug(f'args received: {json.dumps(form_args)}')
docker_image_url = delayed_job_models.get_docker_image_url(job_type)
job_params_only = {
param_key: parameter
for (param_key, parameter) in form_args.items()
}
job_inputs_only = get_job_input_files_desc(file_args)
input_files_hashes = get_input_files_hashes(job_inputs_only)
return submit_job(job_type, job_inputs_only, input_files_hashes,
docker_image_url, job_params_only)
def get_config_for_prop(self, index_name, prop_id):
"""
:param index_name: name of the index to which the property belongs
:param prop_id: full path of the property, such as '_metadata.assay_data.assay_subcellular_fraction'
:return: a dict describing the configuration of a property
"""
cache_key = f'config_for_{index_name}-{prop_id}'
app_logging.debug(f'cache_key: {cache_key}')
cache_response = cache.fail_proof_get(key=cache_key)
if cache_response is not None:
app_logging.debug(f'results were cached')
return cache_response
app_logging.debug(f'results were not cached')
app_logging.debug(
f'getting property config for {prop_id} of index {index_name}')
es_property_description = self.get_property_base_es_description(
index_name, prop_id)
property_override_description = self.get_property_base_override_description(
index_name, prop_id)
config = self.get_merged_prop_config(index_name, prop_id,
es_property_description,
property_override_description)
seconds_valid = RUN_CONFIG.get('es_proxy_cache_seconds')
cache.fail_proof_set(key=cache_key,
value=config,
timeout=seconds_valid)
return config
def get_es_data():
"""
:return: the json response with the data from elasticsearch
"""
form_data = request.form
index_name = sanitise_parameter(form_data.get('index_name'))
raw_es_query = sanitise_parameter(form_data.get('es_query'))
raw_context = sanitise_parameter(form_data.get('context_obj'))
raw_contextual_sort_data = sanitise_parameter(
form_data.get('contextual_sort_data'))
app_logging.debug(f'index_name: {index_name}')
app_logging.debug(f'raw_es_query: {raw_es_query}')
app_logging.debug(f'raw_context: {raw_context}')
app_logging.debug(f'raw_contextual_sort_data: {raw_contextual_sort_data}')
try:
json_response = es_proxy_service.get_es_data(index_name, raw_es_query,
raw_context,
raw_contextual_sort_data)
http_response = jsonify(json_response)
http_cache_utils.add_cache_headers_to_response(http_response)
return http_response
except es_proxy_service.ESProxyServiceError as error:
abort(500, msg=f'Internal server error: {str(error)}')
def prepare_output_dir(job):
"""
Makes sure to create the output dir for the job
:param job: job object for which create the job output
"""
job_output_dir = get_job_output_dir_path(job)
if os.path.exists(job_output_dir):
utils.delete_directory_robustly(job_output_dir)
job.output_dir_path = job_output_dir
delayed_job_models.save_job(job)
os.makedirs(job_output_dir, exist_ok=True)
app_logging.debug(f'Job output dir is {job_output_dir}')
def create_job_run_dir(job):
"""
CReates the directory where the job will run
:param job: job object for which to create the job run directory
"""
job_run_dir = get_job_run_dir(job)
job_input_files_dir = get_job_input_files_dir(job)
if os.path.exists(job_run_dir):
utils.delete_directory_robustly(job_run_dir)
job.run_dir_path = job_run_dir
delayed_job_models.save_job(job)
os.makedirs(job_run_dir, exist_ok=True)
os.makedirs(job_input_files_dir, exist_ok=True)
app_logging.debug(f'Job run dir is {job_run_dir}')
def get_classification_tree():
"""
:return: the go slim target classification tree
"""
cache_key = 'target_classifications_go_slim_1'
app_logging.debug(f'cache_key: {cache_key}')
cache_response = cache.fail_proof_get(key=cache_key)
if cache_response is not None:
app_logging.debug('results are in cache')
return cache_response
tree_generator = GoSlimTreeGenerator()
final_tree = tree_generator.get_classification_tree()
cache_time = int(3.154e7)
cache.fail_proof_set(key=cache_key, value=final_tree, timeout=cache_time)
return final_tree
def create_and_submit_job(job_type, input_files_desc, input_files_hashes,
docker_image_url, job_params):
"""
Creates a job and submits if to LSF
:param job_type: type of job to submit
:param input_files_desc: dict with the paths of the input files
:param input_files_hashes: dict with the hashes of the input files
:param docker_image_url: image of the container to use
:param job_params: parameters of the job
:return: the job object created
"""
job = delayed_job_models.get_or_create(job_type, job_params,
docker_image_url,
input_files_hashes)
job.progress = 0
job.started_at = None
job.finished_at = None
delayed_job_models.save_job(job)
app_logging.debug(f'Submitting Job: {job.id}')
prepare_job_and_submit(job, input_files_desc)
return job
def do_multisearch(body):
"""
:param body: body of the multisearch
:return: the result of the multisearch
"""
cache_key = get_multisearch_cache_key(body)
app_logging.debug(f'cache_key: {cache_key}')
start_time = time.time()
cache_response = cache.fail_proof_get(key=cache_key)
if cache_response is not None:
end_time = time.time()
time_taken = end_time - start_time
app_logging.debug(f'results were cached')
record_that_response_was_cached('multisearch', {'query': body}, time_taken)
return cache_response
app_logging.debug(f'results were not cached')
start_time = time.time()
result = ES.msearch(body=body)
end_time = time.time()
time_taken = end_time - start_time
record_that_response_not_cached('multisearch', {'query': body}, time_taken)
seconds_valid = RUN_CONFIG.get('es_proxy_cache_seconds')
cache.fail_proof_set(key=cache_key, value=result, timeout=seconds_valid)
return result
def parse_search(search_term, es_indexes, selected_es_index):
"""
:param search_term: Term to parse
:param es_indexes: indexes in which the search will be done, separated by a comma
:param selected_es_index: es index to focus on
:return: the query to send to elasticsearch based on the search term provided
"""
cache_key = f'{search_term}-{es_indexes}-{selected_es_index}'
app_logging.debug(f'cache_key: {cache_key}')
cache_response = cache.fail_proof_get(key=cache_key)
if cache_response is not None:
app_logging.debug(f'results were cached')
return cache_response
app_logging.debug(f'results were not cached')
parsed_query = parse_query_str(search_term)
indexes_list = es_indexes.split(',')
best_queries, sorted_indexes_by_score = QueryBuilder.get_best_es_query(
parsed_query, indexes_list, selected_es_index)
response_dict = {
'parsed_query': parsed_query,
'best_es_base_queries': best_queries,
'sorted_indexes_by_score': sorted_indexes_by_score
}
seconds_valid = RUN_CONFIG.get('es_proxy_cache_seconds')
cache.fail_proof_set(key=cache_key,
value=response_dict,
timeout=seconds_valid)
return response_dict
def get_list_of_configured_properties(self, index_name):
"""
:param index_name: the index to check
:return: a list of all the configured properties among all the groups
"""
cache_key = f'facets_configured_properties_for_{index_name}'
app_logging.debug(f'cache_key: {cache_key}')
cache_response = cache.fail_proof_get(key=cache_key)
if cache_response is not None:
app_logging.debug(f'results were cached')
return cache_response
app_logging.debug(f'results were not cached')
with open(self.facets_groups_file_path, 'rt') as groups_file:
groups_config = yaml.load(groups_file, Loader=yaml.FullLoader)
index_groups = groups_config.get(index_name)
if index_groups is None:
raise self.FacetsGroupsConfigurationManagerError(
f'The index {index_name} does not have a configuration set up!'
)
properties_identified = set()
for subgroup in index_groups.values():
for properties_list in subgroup.values():
properties_identified.update(properties_list)
seconds_valid = RUN_CONFIG.get('es_mappings_cache_seconds')
cache.fail_proof_set(key=cache_key,
value=properties_identified,
timeout=seconds_valid)
return list(properties_identified)
def get_context(context_dict):
"""
Returns the context described by the context dict
:param context_dict: dictionary describing the context
:return: the context loaded as an object
"""
context_url = get_context_url(context_dict)
app_logging.debug(f'Loading context from url: {context_url}')
context_request = requests.get(context_url)
if context_request.status_code != 200:
raise ContextLoaderError(
'There was an error while loading the context: ' +
context_request.text)
results = context_request.json()['search_results']
total_results = len(results)
if total_results > WEB_RESULTS_SIZE_LIMIT:
results = results[0:WEB_RESULTS_SIZE_LIMIT]
return results, total_results
def expand_url(url_hash):
"""
:param url_hash: hash of the url to expand
:return: the expanded url corresponding to the hash
"""
raw_document = get_url_shortening(url_hash)
if raw_document is None:
raise URLNotFoundError(f'No url correspond to the hash {url_hash}')
keep_alive = RUN_CONFIG.get('url_shortening').get('keep_alive', False)
if not keep_alive:
expiration_timestamp = raw_document['_source']['expires']
expires = datetime.fromtimestamp(expiration_timestamp / 1000)
else:
expires = extend_expiration_date(raw_document)
app_logging.debug(
f'keep_alive is on, new expiration date is {expires}')
long_url = raw_document['_source']['long_url']
statistics_saver.record_url_was_expanded()
trigger_deletion_of_expired_urls()
return {'long_url': long_url, 'expires': expires}
def get_facets_config_for_group(self, index_name, group_name):
"""
:param index_name: name of the index to which the group belongs
:param group_name: name of the facets group
:return: the configuration for the facets group
"""
cache_key = f'facets_config_for_group_{index_name}-{group_name}_3'
app_logging.debug(f'cache_key: {cache_key}')
cache_response = cache.fail_proof_get(key=cache_key)
if cache_response is not None:
app_logging.debug(f'results were cached')
return cache_response
app_logging.debug(f'results were not cached')
with open(self.facets_groups_file_path, 'rt') as groups_file:
groups_config = yaml.load(groups_file, Loader=yaml.FullLoader)
index_groups = groups_config.get(index_name)
if index_groups is None:
raise self.FacetsGroupsConfigurationManagerError(
f'The index {index_name} does not have a configuration set up!'
)
group_config = index_groups.get(group_name)
if group_config is None:
raise self.FacetsGroupsConfigurationManagerError(
f'The group {group_name} does not exist in index {index_name}!'
)
default_properties = group_config.get('default', {})
optional_properties = group_config.get('optional', {})
config = {
'properties': {
'default':
self.get_facets_config_for_properties(default_properties,
index_name),
'optional':
self.get_facets_config_for_properties(optional_properties,
index_name)
}
}
seconds_valid = RUN_CONFIG.get('es_mappings_cache_seconds')
cache.fail_proof_set(key=cache_key,
value=config,
timeout=seconds_valid)
return config
def save_record_to_elasticsearch(doc, index_name):
dry_run = RUN_CONFIG.get('job_statistics', {}).get('dry_run', False)
es_host = RUN_CONFIG.get('elasticsearch', {}).get('host')
if dry_run:
app_logging.debug(
f'Not actually sending the record to the statistics (dry run): {doc}'
)
else:
app_logging.debug(
f'Sending the following record to the statistics: {doc} '
f'index name: {index_name} es_host: {es_host}')
result = ES.index(index=index_name, body=doc, doc_type='_doc')
app_logging.debug(f'Result {result}')
def shorten_url(long_url):
"""
:param long_url:
:return: an object with the hash and expiration date of the hash
"""
hex_digest = hashlib.md5(long_url.encode('utf-8')).digest()
# replace / and + to avoid routing problems
url_hash = base64.b64encode(hex_digest).decode('utf-8').replace(
'/', '_').replace('+', '-')
app_logging.debug(f'url_hash: {url_hash}')
# check if the url has been shortened before
raw_document = get_url_shortening(url_hash)
already_exists = raw_document is not None
if already_exists:
app_logging.debug(f'already exists')
keep_alive = RUN_CONFIG.get('url_shortening').get('keep_alive', False)
if not keep_alive:
expiration_timestamp = raw_document['_source']['expires']
expires = datetime.fromtimestamp(expiration_timestamp / 1000)
else:
expires = extend_expiration_date(raw_document)
app_logging.debug(
f'keep_alive is on, new expiration date is {expires}')
else:
app_logging.debug(f'Did not exist before')
expires = save_shortened_url(long_url, url_hash)
statistics_saver.record_url_was_shortened()
trigger_deletion_of_expired_urls()
return {'hash': url_hash, 'expires': expires}
def get_config_for_group(self, index_name, group_name):
"""
:param index_name: name of the index
:param group_name: group name as defined in the groups file
:return: the configuration of the group with the following structure:
{
"properties": {
"default": [...], # properties to show by default
"optional:" [...] # properties to show as optional for the user
}
}
"""
cache_key = f'config_for_group_{index_name}-{group_name}'
app_logging.debug(f'cache_key: {cache_key}')
cache_response = cache.fail_proof_get(key=cache_key)
if cache_response is not None:
app_logging.debug(f'results were cached')
return cache_response
app_logging.debug(f'results were not cached')
with open(self.groups_file_path, 'rt') as groups_file:
groups_config = yaml.load(groups_file, Loader=yaml.FullLoader)
index_groups = groups_config.get(index_name, {})
group_config = index_groups.get(group_name)
if group_config is None:
raise self.GroupsConfigurationManagerError(
f'The group {group_name} does not exist in index {index_name}!'
)
props_configs = {}
for sub_group, props_list in group_config.items():
props_configs[sub_group] = self.get_config_for_props_list(
index_name, props_list)
config = {'properties': props_configs}
seconds_valid = RUN_CONFIG.get('es_mappings_cache_seconds')
cache.fail_proof_set(key=cache_key,
value=config,
timeout=seconds_valid)
return config
def get_es_response(index_name, es_query, ignore_cache=False):
"""""
:param index_name: name of the index to query against
:param es_query: dict with the query to send
:param ignore_cache: determines if cache must be ignored or not
:return: the dict with the response from es
"""
cache_key = get_es_query_cache_key(index_name, es_query)
app_logging.debug(f'cache_key: {cache_key}')
start_time = time.time()
if not ignore_cache:
cache_response = cache.fail_proof_get(key=cache_key)
if cache_response is not None:
end_time = time.time()
time_taken = end_time - start_time
app_logging.debug(f'results were cached')
record_that_response_was_cached(index_name, es_query, time_taken)
return cache_response
app_logging.debug(f'results were not cached')
try:
start_time = time.time()
response = ES.search(index=index_name, body=es_query)
end_time = time.time()
time_taken = end_time - start_time
record_that_response_not_cached(index_name, es_query, time_taken)
except elasticsearch.exceptions.RequestError as error:
app_logging.error(f'This query caused an error: ')
app_logging.error(f'index_name:{index_name}')
app_logging.error(f'es_query:')
app_logging.error(es_query)
raise error
seconds_valid = RUN_CONFIG.get('es_proxy_cache_seconds')
if not ignore_cache:
cache.fail_proof_set(key=cache_key, value=response, timeout=seconds_valid)
return response
def get_job_resources_params(job):
"""
Gets the string with the parameters for bsub for the job requirements parameters
:param job: job object for which to generate the parameters string
:return: the parameters if that is the case, empty string if the default settings must be used.
"""
job_config = delayed_job_models.get_job_config(job.type)
source_requirements_script_path = job_config.requirements_script_path
if source_requirements_script_path is None:
return ''
dest_requirements_script_path = Path(
job.run_dir_path).joinpath('requirements_calculation.py')
shutil.copyfile(source_requirements_script_path,
dest_requirements_script_path)
file_stats = os.stat(source_requirements_script_path)
os.chmod(dest_requirements_script_path, file_stats.st_mode | stat.S_IEXEC)
run_params_path = get_job_run_params_file_path(job)
run_command = f'{dest_requirements_script_path} {run_params_path}'
requirements_params_process = subprocess.run(run_command.split(' '),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
return_code = requirements_params_process.returncode
app_logging.debug(f'requirements return code was: {return_code}')
if return_code != 0:
raise JobSubmissionError(
'There was an error when running the job submission script! Please check the logs'
)
app_logging.debug(
f'Run params Output: \n {requirements_params_process.stdout}')
app_logging.debug(
f'Run params Error: \n {requirements_params_process.stderr}')
params_ouput_str = requirements_params_process.stdout.decode().rstrip()
if params_ouput_str == 'DEFAULT':
return ''
return params_ouput_str
def get_es_doc(index_name, doc_id):
"""
:param index_name: name of the intex to which the document belongs
:param doc_id: id of the document
:return: the dict with the response from es corresponding to the document
"""
cache_key = f'document-{doc_id}'
app_logging.debug(f'cache_key: {cache_key}')
equivalent_query = {
"query": {
"ids": {
"values": doc_id
}
}
}
start_time = time.time()
cache_response = cache.fail_proof_get(key=cache_key)
if cache_response is not None:
end_time = time.time()
time_taken = end_time - start_time
app_logging.debug(f'results were cached')
record_that_response_was_cached(index_name, equivalent_query, time_taken)
return cache_response
app_logging.debug(f'results were not cached')
try:
start_time = time.time()
response = ES.get(index=index_name, id=doc_id)
end_time = time.time()
time_taken = end_time - start_time
record_that_response_not_cached(index_name, equivalent_query, time_taken)
except elasticsearch.exceptions.NotFoundError as error:
raise ESDataNotFoundError(repr(error))
seconds_valid = RUN_CONFIG.get('es_proxy_cache_seconds')
cache.fail_proof_set(key=cache_key, value=response, timeout=seconds_valid)
return response
def get_list_of_configured_properties(self, index_name):
"""
:param index_name: the index to check
:return: a list of all the configured properties among all the groups
"""
cache_key = f'configured_properties_for_{index_name}'
app_logging.debug(f'cache_key: {cache_key}')
cache_response = cache.fail_proof_get(key=cache_key)
if cache_response is not None:
app_logging.debug(f'results were cached')
return cache_response
app_logging.debug(f'results were not cached')
with open(self.groups_file_path, 'rt') as groups_file:
groups_config = yaml.load(groups_file, Loader=yaml.FullLoader)
properties_identified = set()
index_groups = groups_config.get(index_name, {})
if index_groups is None:
raise self.GroupsConfigurationManagerError(
f'The index {index_name} does not have a configuration set up!'
)
for subgroup in index_groups.values():
for properties_list in subgroup.values():
for property_id in properties_list:
property_config = self.property_configuration_manager.get_config_for_prop(
index_name, property_id)
is_virtual = property_config.get('is_virtual', False)
# Do not include virtual properties
if is_virtual:
continue
properties_identified.add(property_id)
seconds_valid = RUN_CONFIG.get('es_mappings_cache_seconds')
cache.fail_proof_set(key=cache_key,
value=properties_identified,
timeout=seconds_valid)
return list(properties_identified)
def save_record_to_elasticsearch(doc, index_name):
"""
Saves the record indicated as parameter to the index indicated as parameter
:param doc: doc to save
:param index_name: index where to save the dod
"""
dry_run = RUN_CONFIG.get('usage_statistics', {}).get('dry_run', False)
es_host = RUN_CONFIG.get('usage_statistics', {}).get('elasticsearch', {}).get('host')
es_port = RUN_CONFIG.get('usage_statistics', {}).get('elasticsearch', {}).get('port')
if dry_run:
app_logging.debug(f'Not actually sending the record to the statistics index {index_name} (dry run): {doc}')
else:
app_logging.debug(f'Sending the following record to the statistics: {doc} '
f'index name: {index_name} es_host: {es_host}:{es_port}')
result = ES_MONITORING.index(index=index_name, body=doc, doc_type='_doc')
app_logging.debug(f'Result {result}')
def submit_job(job_type, input_files_desc, input_files_hashes,
docker_image_url, job_params):
"""
Submits job to the queue, and runs it in background
:param job_type: type of job to submit
:param job_params: dict with the job parameters
"""
try:
# See if the job already exists
job = delayed_job_models.get_job_by_params(job_type, job_params,
docker_image_url,
input_files_hashes)
# If it exists, continues here. If not see submits it (see except)
statistics_saver.save_job_cache_record(
job_type=str(job_type),
run_env_type=RUN_CONFIG.get('run_env'),
was_cached=True,
request_date=datetime.utcnow().timestamp() * 1000)
app_logging.debug(f'Job {job.id} already exists, status: {job.status}')
if job.status in [
delayed_job_models.JobStatuses.CREATED,
delayed_job_models.JobStatuses.QUEUED,
delayed_job_models.JobStatuses.RUNNING,
delayed_job_models.JobStatuses.UNKNOWN
]:
return get_job_submission_response(job)
elif job.status == delayed_job_models.JobStatuses.ERROR:
if job.num_failures <= MAX_RETRIES:
app_logging.debug(
f'{job.id} has failed {job.num_failures}. Max retries is {MAX_RETRIES}. '
f'I will submit it again')
job = create_and_submit_job(job_type, input_files_desc,
input_files_hashes,
docker_image_url, job_params)
return get_job_submission_response(job)
else:
app_logging.debug(
f'{job.id} has failed {job.num_failures} times. Max retries is {MAX_RETRIES}. '
f'NOT submitting it again')
return get_job_submission_response(job)
elif job.status == delayed_job_models.JobStatuses.FINISHED:
must_ignore_cache = parse_ignore_cache_param(job_params)
output_was_lost = job_output_was_lost(job)
app_logging.debug(
f'{job.id}: must_ignore_cache: {must_ignore_cache}')
app_logging.debug(f'{job.id}: output_was_lost: {output_was_lost}')
must_resubmit = must_ignore_cache or output_was_lost
app_logging.debug(f'{job.id}: must_resubmit: {must_resubmit}')
if must_resubmit:
app_logging.debug(f'I will delete and submit again {job.id}')
delayed_job_models.delete_job(job)
job = create_and_submit_job(job_type, input_files_desc,
input_files_hashes,
docker_image_url, job_params)
return get_job_submission_response(job)
return get_job_submission_response(job)
except delayed_job_models.JobNotFoundError:
job = create_and_submit_job(job_type, input_files_desc,
input_files_hashes, docker_image_url,
job_params)
statistics_saver.save_job_cache_record(
job_type=str(job_type),
run_env_type=RUN_CONFIG.get('run_env'),
was_cached=False,
request_date=datetime.utcnow().timestamp() * 1000)
return get_job_submission_response(job)
def submit_job_to_lsf(job):
"""
Runs a script that submits the job to LSF
:param job: DelayedJob object
"""
submit_file_path = get_job_submission_script_file_path(job)
submission_output_path = Path(submit_file_path).parent.joinpath(
'submission.out')
submission_error_path = Path(submit_file_path).parent.joinpath(
'submission.err')
lsf_config = RUN_CONFIG.get('lsf_submission')
id_rsa_path = lsf_config['id_rsa_file']
run_command = f'{submit_file_path} {id_rsa_path}'
app_logging.debug(
f'Going to run job submission script, command: {run_command}')
must_run_jobs = RUN_CONFIG.get('run_jobs', True)
if not must_run_jobs:
app_logging.debug(f'Not submitting jobs because run_jobs is False')
return
submission_process = subprocess.run(run_command.split(' '),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
app_logging.debug(f'Submission STD Output: \n {submission_process.stdout}')
app_logging.debug(f'Submission STD Error: \n {submission_process.stderr}')
with open(submission_output_path, 'wb') as submission_out_file:
submission_out_file.write(submission_process.stdout)
with open(submission_error_path, 'wb') as submission_err_file:
submission_err_file.write(submission_process.stderr)
return_code = submission_process.returncode
app_logging.debug(f'submission return code was: {return_code}')
if return_code != 0:
raise JobSubmissionError(
'There was an error when running the job submission script! Please check the logs'
)
lsf_job_id = get_lsf_job_id(str(submission_process.stdout))
job.lsf_job_id = lsf_job_id
job.status = delayed_job_models.JobStatuses.QUEUED
delayed_job_models.save_job(job)
app_logging.debug(f'LSF Job ID is: {lsf_job_id}')
