def __init__(self, project):
"""Encapsulate the dataset access for this ``project`` """
self.project = project
self.sources = {}
self.dhubs = {}
self.DEBUG = settings.DEBUG
self.refdata_model = RefDataManager()
def __init__(self, project):
"""Encapsulate the dataset access for this ``project`` """
self.project = project
self.sources = {}
self.dhubs = {}
self.DEBUG = settings.DEBUG
self.refdata_model = RefDataManager()
def verify_build_platforms(refdata, build_platforms_ref):
build_platforms = refdata.dhub.execute(
proc='refdata_test.selects.test_all_build_platforms', )
build_platforms_set = set()
for build_platform in build_platforms:
build_platforms_set.add(
RefDataManager.get_platform_key(
build_platform.get('os_name'), build_platform.get('platform'),
build_platform.get('architecture')))
assert build_platforms_ref.issubset(build_platforms_set)
def verify_machine_platforms(refdata, machine_platforms_ref):
machine_platforms = refdata.dhub.execute(proc="refdata_test.selects.test_all_machine_platforms")
machine_platforms_set = set()
for machine_platform in machine_platforms:
machine_platforms_set.add(
RefDataManager.get_platform_key(
machine_platform.get("os_name"), machine_platform.get("platform"), machine_platform.get("architecture")
)
)
assert machine_platforms_ref.issubset(machine_platforms_set)
def verify_build_platforms(refdata, build_platforms_ref):
build_platforms = refdata.dhub.execute(proc="refdata_test.selects.test_all_build_platforms")
build_platforms_set = set()
for build_platform in build_platforms:
build_platforms_set.add(
RefDataManager.get_platform_key(
build_platform.get("os_name"), build_platform.get("platform"), build_platform.get("architecture")
)
)
assert build_platforms_ref.issubset(build_platforms_set)
def verify_machine_platforms(refdata, machine_platforms_ref):
machine_platforms = refdata.dhub.execute(
proc='refdata_test.selects.test_all_machine_platforms', )
machine_platforms_set = set()
for machine_platform in machine_platforms:
machine_platforms_set.add(
RefDataManager.get_platform_key(
machine_platform.get('os_name'),
machine_platform.get('platform'),
machine_platform.get('architecture')))
assert machine_platforms_ref.issubset(machine_platforms_set)
class TreeherderModelBase(object):
"""
Base model class for all derived models
"""
logger = logging.getLogger(__name__)
def __init__(self, project):
"""Encapsulate the dataset access for this ``project`` """
self.project = project
self.sources = {}
self.dhubs = {}
self.DEBUG = settings.DEBUG
self.refdata_model = RefDataManager()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.disconnect()
def __str__(self):
"""String representation is project name."""
return self.project
@classmethod
def get_oauth_credentials(cls):
credentials = {}
for source in Datasource.objects.cached():
if (source.contenttype == 'objectstore') and \
source.oauth_consumer_key and \
source.oauth_consumer_secret:
credentials[source.project] = {
'consumer_key': source.oauth_consumer_key,
'consumer_secret': source.oauth_consumer_secret
}
return credentials
def get_dhub(self, contenttype, procs_file_name=None):
"""
The configured datahub for the given contenttype
"""
if not procs_file_name: # pragma: no cover
procs_file_name = "{0}.json".format(contenttype)
if contenttype not in self.dhubs.keys():
datasource = self.get_datasource(contenttype)
self.dhubs[contenttype] = datasource.dhub(procs_file_name)
return self.dhubs[contenttype]
def get_datasource(self, contenttype):
"""The datasource for this contenttype of the project."""
if contenttype not in self.sources.keys():
self.sources[contenttype] = self._get_datasource(contenttype)
return self.sources[contenttype]
def get_inserted_row_ids(self, dhub):
"""
InnoDB guarantees sequential numbers for AUTO INCREMENT when doing
bulk inserts, provided innodb_autoinc_lock_mode is set to 0
(traditional) or 1 (consecutive).
Consequently you can get the first ID from LAST_INSERT_ID() and the
last by adding ROW_COUNT()-1
ref: http://stackoverflow.com/questions/7333524/how-can-i-insert-many-rows-into-a-mysql-table-and-get-ids-back
NOTE: The cursor rowcount is always one for a
INSERT INTO/SELECT FROM DUAL WHERE NOT EXISTS query otherwise
it's equal to the number of rows inserted or updated.
"""
row_count = dhub.connection['master_host']['cursor'].rowcount
ids = []
if row_count > 0:
last_id = dhub.connection['master_host']['cursor'].lastrowid
ids.extend(list(range(last_id - (row_count - 1), last_id + 1)))
return ids
def _process_conditions(self, conditions, allowed_fields=None):
"""Transform a list of conditions into a list of placeholders and
replacement strings to feed a datahub.execute statement."""
placeholders = []
replace_str = ""
if conditions:
for column, condition in conditions.items():
if allowed_fields is None or column in allowed_fields:
if column in allowed_fields:
# we need to get the db column string from the passed
# in querystring column. It could be the same, but
# often it will have a table prefix for the column.
# This allows us to have where clauses on joined fields
# of the query.
column = allowed_fields[column]
for operator, value in condition:
replace_str += "AND {0} {1}".format(column, operator)
if operator == "IN":
# create a list of placeholders of the same length
# as the list of values
replace_str += "({0})".format(",".join(["%s"] *
len(value)))
placeholders += value
else:
replace_str += " %s "
placeholders.append(value)
return replace_str, placeholders
def disconnect(self):
"""Iterate over and disconnect all data sources."""
self.refdata_model.disconnect()
for dhub in self.dhubs.itervalues():
dhub.disconnect()
def _get_datasource(self, contenttype):
"""Find the datasource for this contenttype in the cache."""
try:
return next(source for source in Datasource.objects.cached()
if source.project == self.project
and source.contenttype == contenttype)
except StopIteration:
raise DatasetNotFoundError(self.project, contenttype)
class TreeherderModelBase(object):
"""
Base model class for all derived models
"""
logger = logging.getLogger(__name__)
def __init__(self, project):
"""Encapsulate the dataset access for this ``project`` """
self.project = project
self.sources = {}
self.dhubs = {}
self.DEBUG = settings.DEBUG
self.refdata_model = RefDataManager()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.disconnect()
def __str__(self):
"""String representation is project name."""
return self.project
@classmethod
def get_oauth_credentials(cls):
credentials = {}
for source in Datasource.objects.cached():
if (source.contenttype == 'objectstore') and \
source.oauth_consumer_key and \
source.oauth_consumer_secret:
credentials[source.project] = {
'consumer_key': source.oauth_consumer_key,
'consumer_secret': source.oauth_consumer_secret
}
return credentials
def get_dhub(self, contenttype, procs_file_name=None):
"""
The configured datahub for the given contenttype
"""
if not procs_file_name: # pragma: no cover
procs_file_name = "{0}.json".format(contenttype)
if contenttype not in self.dhubs.keys():
datasource = self.get_datasource(contenttype)
self.dhubs[contenttype] = datasource.dhub(procs_file_name)
return self.dhubs[contenttype]
def get_datasource(self, contenttype):
"""The datasource for this contenttype of the project."""
if contenttype not in self.sources.keys():
self.sources[contenttype] = self._get_datasource(contenttype)
return self.sources[contenttype]
def get_inserted_row_ids(self, dhub):
"""
InnoDB guarantees sequential numbers for AUTO INCREMENT when doing
bulk inserts, provided innodb_autoinc_lock_mode is set to 0
(traditional) or 1 (consecutive).
Consequently you can get the first ID from LAST_INSERT_ID() and the
last by adding ROW_COUNT()-1
ref: http://stackoverflow.com/questions/7333524/how-can-i-insert-many-rows-into-a-mysql-table-and-get-ids-back
NOTE: The cursor rowcount is always one for a
INSERT INTO/SELECT FROM DUAL WHERE NOT EXISTS query otherwise
it's equal to the number of rows inserted or updated.
"""
row_count = dhub.connection['master_host']['cursor'].rowcount
ids = []
if row_count > 0:
last_id = dhub.connection['master_host']['cursor'].lastrowid
ids.extend(list(range(last_id - (row_count - 1), last_id + 1)))
return ids
def disconnect(self):
"""Iterate over and disconnect all data sources."""
self.refdata_model.disconnect()
for dhub in self.dhubs.itervalues():
dhub.disconnect()
def _get_datasource(self, contenttype):
"""Find the datasource for this contenttype in the cache."""
candidate_sources = []
for source in Datasource.objects.cached():
if (source.project == self.project
and source.contenttype == contenttype):
candidate_sources.append(source)
if not candidate_sources:
raise DatasetNotFoundError(self.project, contenttype)
candidate_sources.sort(key=lambda s: s.dataset, reverse=True)
return candidate_sources[0]
class TreeherderModelBase(object):
"""
Base model class for all derived models
"""
def __init__(self, project):
"""Encapsulate the dataset access for this ``project`` """
self.project = project
self.sources = {}
self.dhubs = {}
self.DEBUG = settings.DEBUG
self.refdata_model = RefDataManager()
def __unicode__(self):
"""Unicode representation is project name."""
return self.project
@classmethod
def get_oauth_credentials(cls):
credentials = {}
for source in Datasource.objects.cached():
if (source.contenttype == "objectstore") and source.oauth_consumer_key and source.oauth_consumer_secret:
credentials[source.project] = {
"consumer_key": source.oauth_consumer_key,
"consumer_secret": source.oauth_consumer_secret,
}
return credentials
def get_dhub(self, contenttype, procs_file_name=None):
"""
The configured datahub for the given contenttype
"""
if not procs_file_name: # pragma: no cover
procs_file_name = "{0}.json".format(contenttype)
if not contenttype in self.dhubs.keys():
datasource = self.get_datasource(contenttype)
self.dhubs[contenttype] = datasource.dhub(procs_file_name)
return self.dhubs[contenttype]
def get_datasource(self, contenttype):
"""The datasource for this contenttype of the project."""
if not contenttype in self.sources.keys():
self.sources[contenttype] = self._get_datasource(contenttype)
return self.sources[contenttype]
def get_row_by_id(self, contenttype, table_name, obj_id):
"""
Given an ``id`` get the row for that item.
Return none if not found
"""
proc = "generic.selects.get_row_by_id"
iter_obj = self.get_dhub(contenttype).execute(
proc=proc, replace=[table_name], placeholders=[obj_id], debug_show=self.DEBUG, return_type="iter"
)
return self.as_single(iter_obj, table_name, id=obj_id)
def disconnect(self):
"""Iterate over and disconnect all data sources."""
self.refdata_model.disconnect()
for dhub in self.dhubs.itervalues():
dhub.disconnect()
def _get_datasource(self, contenttype):
"""Find the datasource for this contenttype in the cache."""
candidate_sources = []
for source in Datasource.objects.cached():
if source.project == self.project and source.contenttype == contenttype:
candidate_sources.append(source)
if not candidate_sources:
raise DatasetNotFoundError(self.project, contenttype)
candidate_sources.sort(key=lambda s: s.dataset, reverse=True)
return candidate_sources[0]
class TreeherderModelBase(object):
"""
Base model class for all derived models
"""
logger = logging.getLogger(__name__)
def __init__(self, project):
"""Encapsulate the dataset access for this ``project`` """
self.project = project
self.source = None
self.dhub = None
self.DEBUG = settings.DEBUG
self.refdata_model = RefDataManager()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.disconnect()
def __str__(self):
"""String representation is project name."""
return self.project
def get_dhub(self, procs_file_name=None):
"""
The configured datahub
"""
if not procs_file_name: # pragma: no cover
procs_file_name = "jobs.json"
if not self.dhub:
datasource = self.get_datasource()
self.dhub = datasource.dhub(procs_file_name)
return self.dhub
def get_datasource(self):
"""The datasource of the project."""
if not self.source:
self.source = self._get_datasource()
return self.source
def get_inserted_row_ids(self, dhub):
"""
InnoDB guarantees sequential numbers for AUTO INCREMENT when doing
bulk inserts, provided innodb_autoinc_lock_mode is set to 0
(traditional) or 1 (consecutive).
Consequently you can get the first ID from LAST_INSERT_ID() and the
last by adding ROW_COUNT()-1
ref: http://stackoverflow.com/questions/7333524/how-can-i-insert-many-rows-into-a-mysql-table-and-get-ids-back
NOTE: The cursor rowcount is always one for a
INSERT INTO/SELECT FROM DUAL WHERE NOT EXISTS query otherwise
it's equal to the number of rows inserted or updated.
"""
row_count = dhub.connection['master_host']['cursor'].rowcount
ids = []
if row_count > 0:
last_id = dhub.connection['master_host']['cursor'].lastrowid
ids.extend(
list(range(last_id - (row_count - 1), last_id + 1))
)
return ids
def _process_conditions(self, conditions, allowed_fields=None):
"""Transform a list of conditions into a list of placeholders and
replacement strings to feed a datahub.execute statement."""
placeholders = []
replace_str = ""
if conditions:
for column, condition in conditions.items():
if allowed_fields is None or column in allowed_fields:
if column in allowed_fields:
# we need to get the db column string from the passed
# in querystring column. It could be the same, but
# often it will have a table prefix for the column.
# This allows us to have where clauses on joined fields
# of the query.
column = allowed_fields[column]
for operator, value in condition:
replace_str += "AND {0} {1}".format(column, operator)
if operator == "IN":
# create a list of placeholders of the same length
# as the list of values
replace_str += "({0})".format(
",".join(["%s"] * len(value))
)
placeholders += value
else:
replace_str += " %s "
placeholders.append(value)
return replace_str, placeholders
def disconnect(self):
"""Iterate over and disconnect all data sources."""
self.refdata_model.disconnect()
if self.dhub:
self.dhub.disconnect()
def _get_datasource(self):
"""Find the datasource in the cache."""
try:
return next(source for source in Datasource.objects.cached()
if source.project == self.project)
except StopIteration:
raise DatasetNotFoundError(self.project)
class TreeherderModelBase(object):
"""
Base model class for all derived models
"""
def __init__(self, project):
"""Encapsulate the dataset access for this ``project`` """
self.project = project
self.sources = {}
self.dhubs = {}
self.DEBUG = settings.DEBUG
self.refdata_model = RefDataManager()
def __unicode__(self):
"""Unicode representation is project name."""
return self.project
@classmethod
def get_oauth_credentials(cls):
credentials = {}
for source in Datasource.objects.cached():
if (source.contenttype == 'objectstore') and \
source.oauth_consumer_key and \
source.oauth_consumer_secret:
credentials[source.project] = {
'consumer_key': source.oauth_consumer_key,
'consumer_secret': source.oauth_consumer_secret
}
return credentials
def get_dhub(self, contenttype, procs_file_name=None):
"""
The configured datahub for the given contenttype
"""
if not procs_file_name: # pragma: no cover
procs_file_name = "{0}.json".format(contenttype)
if not contenttype in self.dhubs.keys():
datasource = self.get_datasource(contenttype)
self.dhubs[contenttype] = datasource.dhub(procs_file_name)
return self.dhubs[contenttype]
def get_datasource(self, contenttype):
"""The datasource for this contenttype of the project."""
if not contenttype in self.sources.keys():
self.sources[contenttype] = self._get_datasource(contenttype)
return self.sources[contenttype]
def get_inserted_row_ids(self, dhub):
"""
InnoDB guarantees sequential numbers for AUTO INCREMENT when doing
bulk inserts, provided innodb_autoinc_lock_mode is set to 0
(traditional) or 1 (consecutive).
Consequently you can get the first ID from LAST_INSERT_ID() and the
last by adding ROW_COUNT()-1
ref: http://stackoverflow.com/questions/7333524/how-can-i-insert-many-rows-into-a-mysql-table-and-get-ids-back
NOTE: The cursor rowcount is always one for a
INSERT INTO/SELECT FROM DUAL WHERE NOT EXISTS query otherwise
it's equal to the number of rows inserted or updated.
"""
row_count = dhub.connection['master_host']['cursor'].rowcount
ids = []
if row_count > 0:
last_id = dhub.connection['master_host']['cursor'].lastrowid
ids.extend(list(range(last_id - (row_count - 1), last_id + 1)))
return ids
def submit_publish_to_pulse_tasks(self, ids, data_type):
from treeherder.model.tasks import publish_to_pulse
publish_to_pulse.apply_async(args=[self.project, ids, data_type])
def get_row_by_id(self, contenttype, table_name, obj_id):
"""
Given an ``id`` get the row for that item.
Return none if not found
"""
proc = "generic.selects.get_row_by_id"
iter_obj = self.get_dhub(contenttype).execute(
proc=proc,
replace=[table_name],
placeholders=[obj_id],
debug_show=self.DEBUG,
return_type='iter',
)
return self.as_single(iter_obj, table_name, id=obj_id)
def disconnect(self):
"""Iterate over and disconnect all data sources."""
self.refdata_model.disconnect()
for dhub in self.dhubs.itervalues():
dhub.disconnect()
def _get_datasource(self, contenttype):
"""Find the datasource for this contenttype in the cache."""
candidate_sources = []
for source in Datasource.objects.cached():
if (source.project == self.project
and source.contenttype == contenttype):
candidate_sources.append(source)
if not candidate_sources:
raise DatasetNotFoundError(self.project, contenttype)
candidate_sources.sort(key=lambda s: s.dataset, reverse=True)
return candidate_sources[0]
class TreeherderModelBase(object):
"""
Base model class for all derived models
"""
logger = logging.getLogger(__name__)
def __init__(self, project):
"""Encapsulate the dataset access for this ``project`` """
self.project = project
self.source = None
self.dhub = None
self.DEBUG = settings.DEBUG
self.refdata_model = RefDataManager()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.disconnect()
def __str__(self):
"""String representation is project name."""
return self.project
def get_dhub(self, procs_file_name=None):
"""
The configured datahub
"""
if not procs_file_name: # pragma: no cover
procs_file_name = "jobs.json"
if not self.dhub:
datasource = self.get_datasource()
self.dhub = datasource.dhub(procs_file_name)
return self.dhub
def get_datasource(self):
"""The datasource of the project."""
if not self.source:
self.source = self._get_datasource()
return self.source
def _process_conditions(self, conditions, allowed_fields=None):
"""Transform a list of conditions into a list of placeholders and
replacement strings to feed a datahub.execute statement."""
placeholders = []
replace_str = ""
if conditions:
for column, condition in conditions.items():
if allowed_fields is None or column in allowed_fields:
if allowed_fields and column in allowed_fields:
# we need to get the db column string from the passed
# in querystring column. It could be the same, but
# often it will have a table prefix for the column.
# This allows us to have where clauses on joined fields
# of the query.
column = allowed_fields[column]
for operator, value in condition:
replace_str += "AND {0} {1}".format(column, operator)
if operator == "IN":
# create a list of placeholders of the same length
# as the list of values
replace_str += "({0})".format(
",".join(["%s"] * len(value))
)
placeholders += value
else:
replace_str += " %s "
placeholders.append(value)
return replace_str, placeholders
def disconnect(self):
"""Iterate over and disconnect all data sources."""
self.refdata_model.disconnect()
if self.dhub:
self.dhub.disconnect()
def _get_datasource(self):
"""Find the datasource in the cache."""
try:
return next(source for source in Datasource.objects.cached()
if source.project == self.project)
except StopIteration:
raise DatasetNotFoundError(self.project)
class TreeherderModelBase(object):
"""
Base model class for all derived models
"""
logger = logging.getLogger(__name__)
def __init__(self, project):
"""Encapsulate the dataset access for this ``project`` """
self.project = project
self.source = None
self.dhub = None
self.DEBUG = settings.DEBUG
self.refdata_model = RefDataManager()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.disconnect()
def __str__(self):
"""String representation is project name."""
return self.project
def get_dhub(self, procs_file_name=None):
"""
The configured datahub
"""
if not procs_file_name: # pragma: no cover
procs_file_name = "jobs.json"
if not self.dhub:
datasource = self.get_datasource()
self.dhub = datasource.dhub(procs_file_name)
return self.dhub
def get_datasource(self):
"""The datasource of the project."""
if not self.source:
self.source = self._get_datasource()
return self.source
def _process_conditions(self, conditions, allowed_fields=None):
"""Transform a list of conditions into a list of placeholders and
replacement strings to feed a datahub.execute statement."""
placeholders = []
replace_str = ""
if conditions:
for column, condition in conditions.items():
if allowed_fields is None or column in allowed_fields:
if allowed_fields and column in allowed_fields:
# we need to get the db column string from the passed
# in querystring column. It could be the same, but
# often it will have a table prefix for the column.
# This allows us to have where clauses on joined fields
# of the query.
column = allowed_fields[column]
for operator, value in condition:
replace_str += "AND {0} {1}".format(column, operator)
if operator == "IN":
# create a list of placeholders of the same length
# as the list of values
replace_str += "({0})".format(",".join(["%s"] *
len(value)))
placeholders += value
else:
replace_str += " %s "
placeholders.append(value)
return replace_str, placeholders
def disconnect(self):
"""Iterate over and disconnect all data sources."""
self.refdata_model.disconnect()
if self.dhub:
self.dhub.disconnect()
def _get_datasource(self):
"""Find the datasource in the cache."""
try:
return next(source for source in Datasource.objects.cached()
if source.project == self.project)
except StopIteration:
raise DatasetNotFoundError(self.project)
class TreeherderModelBase(object):
"""
Base model class for all derived models
"""
logger = logging.getLogger(__name__)
def __init__(self, project):
"""Encapsulate the dataset access for this ``project`` """
self.project = project
self.sources = {}
self.dhubs = {}
self.DEBUG = settings.DEBUG
self.refdata_model = RefDataManager()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.disconnect()
def __str__(self):
"""String representation is project name."""
return self.project
@classmethod
def get_oauth_credentials(cls):
credentials = {}
for source in Datasource.objects.cached():
if (source.contenttype == 'objectstore') and \
source.oauth_consumer_key and \
source.oauth_consumer_secret:
credentials[source.project] = {
'consumer_key': source.oauth_consumer_key,
'consumer_secret': source.oauth_consumer_secret
}
return credentials
def get_dhub(self, contenttype, procs_file_name=None):
"""
The configured datahub for the given contenttype
"""
if not procs_file_name: # pragma: no cover
procs_file_name = "{0}.json".format(contenttype)
if contenttype not in self.dhubs.keys():
datasource = self.get_datasource(contenttype)
self.dhubs[contenttype] = datasource.dhub(procs_file_name)
return self.dhubs[contenttype]
def get_datasource(self, contenttype):
"""The datasource for this contenttype of the project."""
if contenttype not in self.sources.keys():
self.sources[contenttype] = self._get_datasource(contenttype)
return self.sources[contenttype]
def get_inserted_row_ids(self, dhub):
"""
InnoDB guarantees sequential numbers for AUTO INCREMENT when doing
bulk inserts, provided innodb_autoinc_lock_mode is set to 0
(traditional) or 1 (consecutive).
Consequently you can get the first ID from LAST_INSERT_ID() and the
last by adding ROW_COUNT()-1
ref: http://stackoverflow.com/questions/7333524/how-can-i-insert-many-rows-into-a-mysql-table-and-get-ids-back
NOTE: The cursor rowcount is always one for a
INSERT INTO/SELECT FROM DUAL WHERE NOT EXISTS query otherwise
it's equal to the number of rows inserted or updated.
"""
row_count = dhub.connection['master_host']['cursor'].rowcount
ids = []
if row_count > 0:
last_id = dhub.connection['master_host']['cursor'].lastrowid
ids.extend(
list(range(last_id - (row_count - 1), last_id + 1))
)
return ids
def disconnect(self):
"""Iterate over and disconnect all data sources."""
self.refdata_model.disconnect()
for dhub in self.dhubs.itervalues():
dhub.disconnect()
def _get_datasource(self, contenttype):
"""Find the datasource for this contenttype in the cache."""
candidate_sources = []
for source in Datasource.objects.cached():
if (source.project == self.project and
source.contenttype == contenttype):
candidate_sources.append(source)
if not candidate_sources:
raise DatasetNotFoundError(self.project, contenttype)
candidate_sources.sort(key=lambda s: s.dataset, reverse=True)
return candidate_sources[0]
class TreeherderModelBase(object):
"""
Base model class for all derived models
"""
def __init__(self, project):
"""Encapsulate the dataset access for this ``project`` """
self.project = project
self.sources = {}
self.dhubs = {}
self.DEBUG = settings.DEBUG
self.refdata_model = RefDataManager()
def __unicode__(self):
"""Unicode representation is project name."""
return self.project
@classmethod
def get_oauth_credentials(cls):
credentials = {}
for source in Datasource.objects.cached():
if (source.contenttype == "objectstore") and source.oauth_consumer_key and source.oauth_consumer_secret:
credentials[source.project] = {
"consumer_key": source.oauth_consumer_key,
"consumer_secret": source.oauth_consumer_secret,
}
return credentials
def get_dhub(self, contenttype, procs_file_name=None):
"""
The configured datahub for the given contenttype
"""
if not procs_file_name: # pragma: no cover
procs_file_name = "{0}.json".format(contenttype)
if not contenttype in self.dhubs.keys():
datasource = self.get_datasource(contenttype)
self.dhubs[contenttype] = datasource.dhub(procs_file_name)
return self.dhubs[contenttype]
def get_datasource(self, contenttype):
"""The datasource for this contenttype of the project."""
if not contenttype in self.sources.keys():
self.sources[contenttype] = self._get_datasource(contenttype)
return self.sources[contenttype]
def get_inserted_row_ids(self, dhub):
"""
InnoDB guarantees sequential numbers for AUTO INCREMENT when doing
bulk inserts, provided innodb_autoinc_lock_mode is set to 0
(traditional) or 1 (consecutive).
Consequently you can get the first ID from LAST_INSERT_ID() and the
last by adding ROW_COUNT()-1
ref: http://stackoverflow.com/questions/7333524/how-can-i-insert-many-rows-into-a-mysql-table-and-get-ids-back
NOTE: The cursor rowcount is always one for a
INSERT INTO/SELECT FROM DUAL WHERE NOT EXISTS query otherwise
it's equal to the number of rows inserted or updated.
"""
row_count = dhub.connection["master_host"]["cursor"].rowcount
ids = []
if row_count > 0:
last_id = dhub.connection["master_host"]["cursor"].lastrowid
ids.extend(list(range(last_id - (row_count - 1), last_id + 1)))
return ids
def submit_publish_to_pulse_tasks(self, ids, data_type):
from treeherder.model.tasks import publish_to_pulse
publish_to_pulse.apply_async(args=[self.project, ids, data_type])
def get_row_by_id(self, contenttype, table_name, obj_id):
"""
Given an ``id`` get the row for that item.
Return none if not found
"""
proc = "generic.selects.get_row_by_id"
iter_obj = self.get_dhub(contenttype).execute(
proc=proc, replace=[table_name], placeholders=[obj_id], debug_show=self.DEBUG, return_type="iter"
)
return self.as_single(iter_obj, table_name, id=obj_id)
def disconnect(self):
"""Iterate over and disconnect all data sources."""
self.refdata_model.disconnect()
for dhub in self.dhubs.itervalues():
dhub.disconnect()
def _get_datasource(self, contenttype):
"""Find the datasource for this contenttype in the cache."""
candidate_sources = []
for source in Datasource.objects.cached():
if source.project == self.project and source.contenttype == contenttype:
candidate_sources.append(source)
if not candidate_sources:
raise DatasetNotFoundError(self.project, contenttype)
candidate_sources.sort(key=lambda s: s.dataset, reverse=True)
return candidate_sources[0]
def do_job_ingestion(jm, refdata, job_data, sample_resultset, verify_data=True):
"""
Ingest ``job_data`` which will be JSON job blobs.
``verify_data`` - whether or not to run the ingested jobs
through the verifier.
"""
jm.store_result_set_data(sample_resultset)
max_index = len(sample_resultset) - 1
resultset_index = 0
# Structures to test if we stored everything
build_platforms_ref = set()
machine_platforms_ref = set()
machines_ref = set()
options_ref = set()
job_types_ref = set()
products_ref = set()
result_sets_ref = set()
log_urls_ref = set()
coalesced_job_guids = {}
coalesced_replacements = []
artifacts_ref = {}
blobs = []
for index, blob in enumerate(job_data):
if resultset_index > max_index:
resultset_index = 0
# Modify job structure to sync with the resultset sample data
if 'sources' in blob:
del blob['sources']
blob['revision_hash'] = sample_resultset[resultset_index]['revision_hash']
blobs.append(blob)
resultset_index += 1
# Build data structures to confirm everything is stored
# as expected
if verify_data:
job_guid = blob['job']['job_guid']
job = blob['job']
build_platforms_ref.add(
RefDataManager.get_platform_key(
job.get('build_platform', {}).get('os_name', 'unkown'),
job.get('build_platform', {}).get('platform', 'unkown'),
job.get('build_platform', {}).get('architecture', 'unknown')
))
machine_platforms_ref.add(
RefDataManager.get_platform_key(
job.get('machine_platform', {}).get('os_name', 'unkown'),
job.get('machine_platform', {}).get('platform', 'unkown'),
job.get('machine_platform', {}).get('architecture', 'unknown')
))
machines_ref.add(job.get('machine', 'unknown'))
options_ref = options_ref.union(job.get('option_collection', []).keys())
job_types_ref.add(job.get('name', 'unknown'))
products_ref.add(job.get('product_name', 'unknown'))
result_sets_ref.add(blob['revision_hash'])
log_url_list = job.get('log_references', [])
for log_data in log_url_list:
log_urls_ref.add(log_data['url'])
artifact_name = job.get('artifact', {}).get('name')
if artifact_name:
artifacts_ref[artifact_name] = job.get('artifact')
coalesced = blob.get('coalesced', [])
if coalesced:
coalesced_job_guids[job_guid] = coalesced
coalesced_replacements.append('%s')
# Store the modified json blobs
jm.store_job_data(blobs)
if verify_data:
# Confirms stored data matches whats in the reference data structs
verify_build_platforms(refdata, build_platforms_ref)
verify_machine_platforms(refdata, machine_platforms_ref)
verify_machines(refdata, machines_ref)
verify_options(refdata, options_ref)
verify_job_types(refdata, job_types_ref)
verify_products(refdata, products_ref)
verify_result_sets(jm, result_sets_ref)
verify_log_urls(jm, log_urls_ref)
verify_artifacts(jm, artifacts_ref)
verify_coalesced(jm, coalesced_job_guids, coalesced_replacements)
def do_job_ingestion(jm, refdata, job_data, sample_resultset, verify_data=True):
"""
Ingest ``job_data`` which will be JSON job blobs.
``verify_data`` - whether or not to run the ingested jobs
through the verifier.
"""
jm.store_result_set_data(sample_resultset)
max_index = len(sample_resultset) - 1
resultset_index = 0
# Structures to test if we stored everything
build_platforms_ref = set()
machine_platforms_ref = set()
machines_ref = set()
options_ref = set()
job_types_ref = set()
products_ref = set()
result_sets_ref = set()
log_urls_ref = set()
coalesced_job_guids = {}
coalesced_replacements = []
artifacts_ref = {}
blobs = []
for index, blob in enumerate(job_data):
if resultset_index > max_index:
resultset_index = 0
# Modify job structure to sync with the resultset sample data
if "sources" in blob:
del blob["sources"]
blob["revision_hash"] = sample_resultset[resultset_index]["revision_hash"]
blobs.append(blob)
resultset_index += 1
# Build data structures to confirm everything is stored
# as expected
if verify_data:
job_guid = blob["job"]["job_guid"]
job = blob["job"]
build_platforms_ref.add(
RefDataManager.get_platform_key(
job.get("build_platform", {}).get("os_name", "unkown"),
job.get("build_platform", {}).get("platform", "unkown"),
job.get("build_platform", {}).get("architecture", "unknown"),
)
)
machine_platforms_ref.add(
RefDataManager.get_platform_key(
job.get("machine_platform", {}).get("os_name", "unkown"),
job.get("machine_platform", {}).get("platform", "unkown"),
job.get("machine_platform", {}).get("architecture", "unknown"),
)
)
machines_ref.add(job.get("machine", "unknown"))
options_ref = options_ref.union(job.get("option_collection", []).keys())
job_types_ref.add(job.get("name", "unknown"))
products_ref.add(job.get("product_name", "unknown"))
result_sets_ref.add(blob["revision_hash"])
log_url_list = job.get("log_references", [])
for log_data in log_url_list:
log_urls_ref.add(log_data["url"])
artifact_name = job.get("artifact", {}).get("name")
if artifact_name:
artifacts_ref[artifact_name] = job.get("artifact")
coalesced = blob.get("coalesced", [])
if coalesced:
coalesced_job_guids[job_guid] = coalesced
coalesced_replacements.append("%s")
# Store the modified json blobs
jm.store_job_data(blobs)
# Process the job objects in chunks of size == process_objects_limit
process_objects_limit = 1000
chunks = grouper(job_data, process_objects_limit)
for c in chunks:
jm.process_objects(process_objects_limit, raise_errors=True)
if verify_data:
# Confirms stored data matches whats in the reference data structs
verify_build_platforms(refdata, build_platforms_ref)
verify_machine_platforms(refdata, machine_platforms_ref)
verify_machines(refdata, machines_ref)
verify_options(refdata, options_ref)
verify_job_types(refdata, job_types_ref)
verify_products(refdata, products_ref)
verify_result_sets(jm, result_sets_ref)
verify_log_urls(jm, log_urls_ref)
verify_artifacts(jm, artifacts_ref)
verify_coalesced(jm, coalesced_job_guids, coalesced_replacements)
# Default verification confirms we loaded all of the objects
complete_count = jm.get_os_dhub().execute(proc="objectstore_test.counts.complete")[0]["complete_count"]
loading_count = jm.get_os_dhub().execute(proc="objectstore_test.counts.loading")[0]["loading_count"]
assert complete_count == len(job_data)
assert loading_count == 0
