def read(self, query, log=None):
self.items = []
for item in self.do_query(query):
new_item = self._empty_item()
new_item['key'] = item.key
new_item['summary'] = Jira.strip_non_ascii(
Jira.remove_version_and_platform(item.fields.summary))
new_item['description'] = Jira.strip_non_ascii(
item.fields.description)
new_item['item'] = item
self.items.append(new_item)
try:
self._add_to_dict(self.items_by_key, new_item['key'], new_item,
"key")
except KeyError as e:
pass
# log.logger.warning("%s", e)
try:
self._add_to_dict(self.items_by_summary, new_item['summary'],
new_item, "summary")
except KeyError as e:
pass
# log.logger.warning("%s", e)
try:
self._add_to_dict(self.items_by_description,
new_item['description'], new_item,
"description")
except KeyError as e:
pass
def read_items(query, log=None):
"""Read items into summary based dictionary, warning on duplicates"""
dictionary = {}
for item in jira.do_query(query):
item_key = Jira.remove_version_and_platform(Jira.strip_non_ascii(item.fields.summary))
if item_key not in dictionary:
dictionary[item_key] = [item]
else:
# So, what we have now is a POTENTIAL duplicate. figure out if it really is.
if item.key != dictionary[item_key][0].key:
# Yep, it's not the same item key...
dictionary[item_key].append(item)
log.logger.debug("Item key '%s' : '%s' creates a duplicate entry with key '%s': '%s'",
item.key, item.fields.summary,
dictionary[item_key][0].key, dictionary[item_key][0].fields.summary)
pass
return dictionary
def compare_items(item_kind, source_name, source_query, target_name, target_query, log=None):
def read_items(query, log=None):
"""Read items into summary based dictionary, warning on duplicates"""
dictionary = {}
for item in jira.do_query(query):
item_key = Jira.remove_version_and_platform(Jira.strip_non_ascii(item.fields.summary))
if item_key not in dictionary:
dictionary[item_key] = [item]
else:
# So, what we have now is a POTENTIAL duplicate. figure out if it really is.
if item.key != dictionary[item_key][0].key:
# Yep, it's not the same item key...
dictionary[item_key].append(item)
log.logger.debug("Item key '%s' : '%s' creates a duplicate entry with key '%s': '%s'",
item.key, item.fields.summary,
dictionary[item_key][0].key, dictionary[item_key][0].fields.summary)
pass
return dictionary
def scan_dups(source_dict, printit):
for k, v in source_dict.items():
if len(v) > 1:
keys = []
for item in v:
keys.append(item.key)
printit(keys, k)
return
source = read_items(source_query, log)
scan_dups(source, lambda x, y: log.logger.error("Duplicate %s summaries: %s '%s'", source_name, x, y))
log.logger.info( "Source has %d items in dictionary", len(source))
target = read_items(target_query, log)
scan_dups(target, lambda x, y: log.logger.error("Duplicate %s summaries: %s '%s'", target_name, x, y))
log.logger.info( "Target has %d items in dictionary", len(target))
# -- Everything in source should be copied to target:
not_in_target = [{'source': value[0].key, 'summary': key}
for key, value in source.items()
if Jira.remove_version_and_platform(Jira.strip_non_ascii(key)) not in target]
if len(not_in_target) > 0:
log.logger.error("")
log.logger.error("Could not find %s %s (source) %s summary items in target: ",
len(not_in_target), source_name, item_kind)
log.logger.error("")
for item in not_in_target:
log.logger.error("Source '%s', summary text: '%s'", item['source'], item['summary'])
log.logger.error("--")
# # -- Target should not have stuff in it that's not from the source!:
not_in_source = [{'target': value[0].key, 'summary': Jira.remove_version_and_platform(Jira.strip_non_ascii(key))}
for key, value in target.items()
if Jira.remove_version_and_platform(Jira.strip_non_ascii(key)) not in source]
if len(not_in_source) > 0:
log.logger.error("")
log.logger.error("Could not find %s %s (target) %s summary items in source: ",
len(not_in_source), target_name, item_kind)
log.logger.error("")
for item in not_in_source:
log.logger.error("%s Target '%s', summary text: '%s'", item_kind, item['target'], item['summary'])
log.logger.error("--")
return
def copy_platform_to_platform(parser, scenario, config, queries, search, log=None):
"""Copy platform to platform, based on the UCIS and E-Feature entries of the source platform"""
#
# -- Some sleight of hand here... Original code iterates the results of a JQL query.
# If xls_source, this will read in an XLS file, look for a column named "Key" and
# read those key values from Jira as source items.
#
if 'xls_source' in scenario:
import pandas as pd
source_file = realpath(dirname(realpath(sys.argv[0])) + '/../' + scenario['xls_source'])
xls_data_frame = pd.read_excel(source_file, sheetname=0)
def preq_item_list(jira):
for i, item in xls_data_frame.iterrows():
key = item['Key']
if key.upper().startswith('PREQ'):
yield jira.issue(key), item
def areq_e_feature_list(jira):
for i, item in xls_data_frame.iterrows():
key = item['Key']
if key.upper().startswith('AREQ'):
yield jira.issue(key), item
else:
preq_source_query = get_query('preq_source_query', queries, copy_platform_to_platform.__name__, params=scenario, log=log)
if preq_source_query is not None:
def preq_item_list(jira):
for preq_item in jira.do_query(preq_source_query):
yield preq_item, None
areq_source_e_feature_query = get_query('areq_source_e_feature', queries, copy_platform_to_platform.__name__,
params=scenario, log=log)
if areq_source_e_feature_query is not None:
def areq_e_feature_list(jira):
for e_feature in jira.do_query(areq_source_e_feature_query):
yield e_feature, None
preq_target_query = get_query('preq_target_query', queries, copy_platform_to_platform.__name__, params=scenario, log=log)
areq_target_e_feature_query = get_query('areq_target_e_feature', queries, copy_platform_to_platform.__name__, params=scenario, log=log)
target_feature_query = get_query('target_feature_query', queries, copy_platform_to_platform.__name__, params=scenario, log=log)
target_summary_format = get_query('target_summary_format', queries, copy_platform_to_platform.__name__, params=scenario, log=log)
log.logger.info("Examining source platform {splatform}, source android version {sversion}, target android version {tversion}".format_map(scenario))
verify = scenario['verify']
update = scenario['update']
verify_copy = scenario['verify_copy'] if 'verify_copy' in scenario else True;
log.logger.info("Verify is %s and Update is %s", verify, update)
log.logger.info("=================================================================")
# -- Get and format it:
jira = Jira(scenario['name'], search, log=log.logger)
global_id = jira.get_field_name("Global ID")
feature_id = jira.get_field_name("Feature ID")
source_preq_scanned = 0
source_areq_scanned = 0
ucis_created = 0
e_features_created = 0
warnings_issued = 0
verify_failures = 0
update_failures = 0
processing_errors = 0
update_count = 0
preq_count = 0
areq_count = 0
def compare_items(item_kind, source_name, source_query, target_name, target_query, log=None):
def read_items(query, log=None):
"""Read items into summary based dictionary, warning on duplicates"""
dictionary = {}
for item in jira.do_query(query):
item_key = Jira.remove_version_and_platform(Jira.strip_non_ascii(item.fields.summary))
if item_key not in dictionary:
dictionary[item_key] = [item]
else:
# So, what we have now is a POTENTIAL duplicate. figure out if it really is.
if item.key != dictionary[item_key][0].key:
# Yep, it's not the same item key...
dictionary[item_key].append(item)
log.logger.debug("Item key '%s' : '%s' creates a duplicate entry with key '%s': '%s'",
item.key, item.fields.summary,
dictionary[item_key][0].key, dictionary[item_key][0].fields.summary)
pass
return dictionary
def scan_dups(source_dict, printit):
for k, v in source_dict.items():
if len(v) > 1:
keys = []
for item in v:
keys.append(item.key)
printit(keys, k)
return
source = read_items(source_query, log)
scan_dups(source, lambda x, y: log.logger.error("Duplicate %s summaries: %s '%s'", source_name, x, y))
log.logger.info( "Source has %d items in dictionary", len(source))
target = read_items(target_query, log)
scan_dups(target, lambda x, y: log.logger.error("Duplicate %s summaries: %s '%s'", target_name, x, y))
log.logger.info( "Target has %d items in dictionary", len(target))
# -- Everything in source should be copied to target:
not_in_target = [{'source': value[0].key, 'summary': key}
for key, value in source.items()
if Jira.remove_version_and_platform(Jira.strip_non_ascii(key)) not in target]
if len(not_in_target) > 0:
log.logger.error("")
log.logger.error("Could not find %s %s (source) %s summary items in target: ",
len(not_in_target), source_name, item_kind)
log.logger.error("")
for item in not_in_target:
log.logger.error("Source '%s', summary text: '%s'", item['source'], item['summary'])
log.logger.error("--")
# # -- Target should not have stuff in it that's not from the source!:
not_in_source = [{'target': value[0].key, 'summary': Jira.remove_version_and_platform(Jira.strip_non_ascii(key))}
for key, value in target.items()
if Jira.remove_version_and_platform(Jira.strip_non_ascii(key)) not in source]
if len(not_in_source) > 0:
log.logger.error("")
log.logger.error("Could not find %s %s (target) %s summary items in source: ",
len(not_in_source), target_name, item_kind)
log.logger.error("")
for item in not_in_source:
log.logger.error("%s Target '%s', summary text: '%s'", item_kind, item['target'], item['summary'])
log.logger.error("--")
return
# -- Copy source preqs to target:
# (Get the list of already existing PREQs for this platform and version!)
if 'copy_preq' not in scenario or scenario['copy_preq']: # e.g., copy_preq is undefined or copy_preq = True
for source_preq, data_frame in preq_item_list(jira):
preq_count += 1
updated = False
# # -- Remove old version and platform, prepend new version and platform
source_preq_scanned += 1
log.logger.debug("Search for: '%s'", source_preq.fields.summary)
target_summary = Jira.remove_version_and_platform(source_preq.fields.summary)
target_summary = target_summary_format % target_summary
existing_preq = jira.get_item(preq_summary=target_summary, log=log)
if existing_preq is not None:
# -- This is good, PREQ is already there so nothing to do.
log.logger.info("%s Found existing UCIS: %s '%s'",
preq_count, existing_preq.key, existing_preq.fields.summary)
# -- Note: Patch the GID entry of this item...
if 'FIX_GID' in scenario and scenario['FIX_GID']:
update_fields = {}
if getattr(existing_preq.fields, global_id) is None or not getattr(existing_preq.fields, global_id):
# -- Patch the GID entry of this item...
log.logger.info("GID of %s is empty, should be %s from %s",
existing_preq.key, getattr(source_preq.fields, global_id), source_preq.key)
update_fields[global_id] = getattr(source_preq.fields, global_id)
if getattr(existing_preq.fields, feature_id) is None or not getattr(existing_preq.fields, feature_id):
# -- Patch the Feature ID entry of this item...
log.logger.info("Feature ID of %s is empty, should be %s from %s",
existing_preq.key, getattr(source_preq.fields, feature_id), source_preq.key)
update_fields[feature_id] = getattr(source_preq.fields, feature_id)
if update and update_fields:
existing_preq.update(notify=False, fields=update_fields)
updated = True
#
# Note that because of where it is, it only affects PREQs, and we want both...
#
if 'UPDATE_FIELDS' in scenario and scenario['UPDATE_FIELDS']:
count = update_fields_and_link(jira, source_preq, existing_preq, update, 0, scenario,
log=log, data_frame=locals())
if count != 0:
updated = True
if update and 'UPDATE_STATUS' in scenario and scenario['UPDATE_STATUS']:
if set_e_feature_status(jira, source_preq, existing_preq, log, scenario):
updated = True
# ===================================================================================================
pass
else:
# -- This Target PREQ is missing, so use Source preq as template to create a new UCIS for the platform:
log.logger.debug("Need to create new UCIS for: '%s'", target_summary)
if update and ('CREATE_MISSING_UCIS' not in scenario or scenario['CREATE_MISSING_UCIS']):
# -- Create a new UCIS(!) PREQ
result = jira.create_ucis(target_summary, source_preq, scenario, log=log, data_frame=locals())
log.logger.info("%s Created a new UCIS %s for %s", areq_count, result.key, target_summary)
updated = True
ucis_created += 1
if update and 'UPDATE_STATUS' in scenario and scenario['UPDATE_STATUS']:
if set_ucis_status(jira, source_preq, result, log, scenario):
updated = True
if 'UPDATE_FIELDS' in scenario and scenario['UPDATE_FIELDS']:
count = update_fields_and_link(jira, source_preq, result, update, 0, scenario,
log=log, data_frame=locals())
if count != 0:
updated = True
else:
log.logger.warning("Target UCIS is missing, sourced from %s: '%s'", source_preq.key, target_summary)
warnings_issued += 1
if updated:
update_count += 1
if scenario['createmax'] and update_count>=scenario['createmax']:
break
pass
update_count = 0
# -- copy source e-features to output
# This keeps having an exception because the total number of items seems to be changing...
if 'copy_areq' not in scenario or scenario['copy_areq']: # e.g., copy_areq is undefined or copy_areq = True
# features = [feature for feature in areq_e_feature_list(jira)]
# for source_e_feature in features:
for source_e_feature, data_frame in areq_e_feature_list(jira):
areq_count += 1
updated = False
# -- The parent for this one should already be in source_features
source_areq_scanned += 1
lookup = source_e_feature.fields.parent.key
try:
# todo: This could actually just be:
parent_feature = jira.issue(lookup)
# parent_feature = jira.get_item(key=lookup, log=log)
except Exception as e:
parent_feature = None # This should never happen!
log.logger.fatal("%s: Could not locate parent %s of E-Feature %s, looked for '%s'. continuing",
e, source_e_feature.fields.parent.key, source_e_feature.key, lookup)
# -- Well, if we couldn't find the parent, we can't continue
warnings_issued += 1
continue
# -- OK, at this point we can create the E-Feature record, if it's not going to be a duplicate...
target_summary = Jira.remove_version_and_platform(source_e_feature.fields.summary).strip()
target_summary = target_summary_format % target_summary
existing_feature = jira.get_item(areq_summary=target_summary, log=log)
if existing_feature is not None:
# -- This E-Feature already exists, don't touch it!
log.logger.info("%s The targeted E-Feature '%s' already exists! %s, %s: %s",
areq_count, target_summary, source_e_feature.key, existing_feature.key, existing_feature.fields.summary)
if 'UPDATE_FIELDS' in scenario and scenario['UPDATE_FIELDS']:
count = update_fields_and_link(jira, source_e_feature, existing_feature, update, 0, scenario,
log=log, data_frame=locals())
if count != 0:
updated = True
if update and 'UPDATE_STATUS' in scenario and scenario['UPDATE_STATUS']:
if set_e_feature_status(jira, source_e_feature, existing_feature, log, scenario):
updated = True
else:
if update:
log.logger.info("%s Creating a new E-Feature for Feature %s: %s",
areq_count, parent_feature.key, target_summary)
if 'clone_from_sibling' in scenario and scenario['clone_from_sibling']:
created_e_feature = jira.clone_e_feature_from_e_feature(target_summary, parent_feature,
source_e_feature, scenario,
log=log, data_frame=locals())
else:
created_e_feature = jira.clone_e_feature_from_parent(target_summary, parent_feature,
scenario, sibling=source_e_feature,
log=log, data_frame=locals())
updated = True
e_features_created += 1
if 'UPDATE_FIELDS' in scenario and scenario['UPDATE_FIELDS']:
count = update_fields_and_link(jira, source_e_feature, created_e_feature, update,
0, scenario, log=log, data_frame=locals())
if count != 0:
updated = True
if update and 'UPDATE_STATUS' in scenario and scenario['UPDATE_STATUS']:
if set_e_feature_status(jira, source_e_feature, created_e_feature, log, scenario):
updated = True
else:
log.logger.info("%s Target E-Feature is missing for Source E-Feature %s, Feature %s: '%s'",
areq_count, source_e_feature.key, parent_feature.key, target_summary)
# -- Create a new E-Feature(!) PREQ
if updated:
update_count += 1
if scenario['createmax'] and update_count>=scenario['createmax']:
break
# -- TODO: Need to account for source and target version and platform
if verify_copy:
compare_items("UCIS", scenario['splatform'], preq_source_query, scenario['tplatform'], preq_target_query, log=log)
compare_items("E-Feature", scenario['splatform'], areq_source_e_feature_query, scenario['tplatform'], areq_target_e_feature_query, log=log)
else:
log.logger.warning("Not checking that copy was complete or that duplicates were created.")
log.logger.info("-----------------------------------------------------------------")
log.logger.info("%s UCIS source entries were considered. ", source_preq_scanned)
log.logger.info("%s target UCIS entries were created. ", ucis_created)
log.logger.info("%s E-Feature source entries were considered. ", source_areq_scanned)
log.logger.info("%s target E-Features entries were created. ", e_features_created)
log.logger.info("%s warnings were issued. ", warnings_issued)
log.logger.info("")
# if verify:
# log.logger.info("%s E-Feature comparison failure(s). ", verify_failures)
#
# if update:
# log.logger.info("%s new E-Feature(s) were created, %s update failures. ", update_count, update_failures)
log.logger.info("%s processing error(s). ", processing_errors)
