def transfer_metadata(self, finfo, local_root, dest):
# TODO this can't be hard-coded for thumbs! generalize
# transfer thumbnail info if exists
if 'parent_full' in finfo:
parent_full_path, parent_name = os.path.split(finfo['parent_full'])
rel_path_parent = u.make_rel_path(local_root, parent_full_path, strict=True)
dest['parent_key'] = u.make_key(rel_path_parent, parent_name)
if 'thumb_full' in finfo:
thumb_full_path, thumb_name = os.path.split(finfo['thumb_full'])
rel_path_thumb = u.make_rel_path(local_root, thumb_full_path, strict=True)
dest['thumb_key'] = u.make_key(rel_path_thumb, thumb_name)
# copy other useful metadata
if 'width' in finfo and 'height' in finfo:
dest['width'] = finfo['width']
dest['height'] = finfo['height']
def parse_alert_info(desc):
'''
Parse all the fields in an issue's description and return
them as a tuple. If parsing fails for one of the fields,
return a tuple of None's.
'''
failed = None, None, None, None
m = re.search('REPOSITORY_NAME=(.*)$', desc, re.MULTILINE)
if m is None:
return failed
repo_id = m.group(1)
m = re.search('ALERT_NUMBER=(.*)$', desc, re.MULTILINE)
if m is None:
return failed
alert_num = int(m.group(1))
m = re.search('REPOSITORY_KEY=(.*)$', desc, re.MULTILINE)
if m is None:
return failed
repo_key = m.group(1)
m = re.search('ALERT_KEY=(.*)$', desc, re.MULTILINE)
if m is None:
return failed
alert_key = m.group(1)
# consistency checks:
if repo_key != util.make_key(repo_id) \
or alert_key != util.make_alert_key(repo_id, alert_num):
return failed
return repo_id, alert_num, repo_key, alert_key
def create_issue(self, repo_id, rule_id, rule_desc, alert_url, alert_num):
raw = self.j.create_issue(
project=self.projectkey,
summary='{prefix} {rule} in {repo}'.format(
prefix=TITLE_PREFIX,
rule=rule_id,
repo=repo_id
),
description=DESC_TEMPLATE.format(
rule_desc=rule_desc,
alert_url=alert_url,
repo_id=repo_id,
alert_num=alert_num,
repo_key=util.make_key(repo_id),
alert_key=util.make_alert_key(repo_id, alert_num)
),
issuetype={'name': 'Bug'}
)
logger.info('Created issue {issue_key} for alert {alert_num} in {repo_id}.'.format(
issue_key=raw.key,
alert_num=alert_num,
repo_id=repo_id
))
return JiraIssue(self, raw)
def fetch_issues(self, repo_id, alert_num=None):
if alert_num is None:
key = util.make_key(repo_id)
else:
key = util.make_alert_key(repo_id, alert_num)
issue_search = 'project={jira_project} and description ~ "{key}"'.format(
jira_project='\"{}\"'.format(self.projectkey),
key=key
)
issues = list(filter(lambda i: i.is_managed(), [JiraIssue(self, raw) for raw in self.j.search_issues(issue_search, maxResults=0)]))
logger.debug('Search {search} returned {num_results} results.'.format(
search=issue_search,
num_results=len(issues)
))
return issues
def get_tree_info(self, bucket=None, remote_root=''):
self.tree_info.clear()
for dir_name, subdirs, files in os.walk(self._file_dest_root):
for file_name in files:
full_src = dir_name + '/' + file_name
rel_path = u.make_rel_path(self._file_dest_root, dir_name, strict=False, no_leading_slash=True)
local_meta = u.local_metadata(dir_name, file_name)
local_meta['key'] = u.make_key(rel_path, file_name)
local_meta['md5'] = u.md5(full_src)
# important: must never expose 'full' outside this class - it's a private
# implementation detail. Same for 'path'. Only 'key' is public
del local_meta['full']
del local_meta['path']
self.tree_info[local_meta['key']] = local_meta
def default_template_file_action(self,
dir_name,
file_name,
dest_rel_path=None,
dest_name=None):
template_full = dir_name + '/' + file_name
Config.log("default_template_file_action '%s'" % template_full,
tag='DEFAULT_TEMPLATE_FILE_ACTION')
if dest_name:
rel_path = dest_rel_path
dest_path = u.pathify(self.output_root, dest_rel_path)
else:
rel_path = u.make_rel_path(self.site_root, dir_name)
dest_path = u.pathify(self.output_root, rel_path)
dest_name = file_name
u.ensure_path(dest_path)
dest_full = u.pathify(dest_path, dest_name)
info = {
'name': dest_name,
'path': dest_path,
'rel_path': rel_path,
'full': dest_full,
'key': u.make_key(rel_path, dest_name)
}
if self.config.is_template_type(file_name):
template = open(template_full).read()
output = u.debracket(template, self.interpret)
if not self.config.is_special_file(info['key']):
open(dest_full, 'w').write(output)
local = u.local_metadata(dest_path, dest_name)
info['size'] = local['size']
info['modified'] = local['modified']
info['md5'] = u.md5(dest_full)
self.track_file(info)
else:
shutil.copyfile(template_full, dest_full)
local = u.local_metadata(dest_path, dest_name)
info['size'] = local['size']
info['modified'] = local['modified']
info['md5'] = u.md5(dest_full)
self.track_file(info)
def track_file(self, finfo):
full = finfo['full']
Config.log(full, tag='TP_TRACK_FILE')
if 'md5' not in finfo:
finfo['md5'] = u.md5(finfo['full'])
if full in self.file_info:
# don't replace finfo unless file has actually changed
old_finfo = self.file_info[full]
if finfo['md5'] == old_finfo['md5']:
Config.log(full, tag='TP_TRACK_FILE_UNCHANGED')
return
self.file_info[full] = finfo
if self._track_file_callback and self.will_upload(finfo['full']):
if 'rel_path' not in finfo:
finfo['rel_path'] = u.make_rel_path(self.config.output, finfo['path'], no_leading_slash=True)
if 'key' not in finfo:
finfo['key'] = u.make_key(finfo['rel_path'], finfo['name'])
if ('size' not in finfo) or ('modified' not in finfo):
logging.error('track_file (%s): finfo missing size and/or modified' % finfo['full'])
tmp = u.local_metadata(finfo['path'], finfo['name'])
finfo['size'] = tmp['size']
finfo['modified'] = tmp['modified']
self._track_file_callback(finfo)
{rule_desc}
{alert_url}
----
This issue was automatically generated from a GitHub alert, and will be automatically resolved once the underlying problem is fixed.
DO NOT MODIFY DESCRIPTION BELOW LINE.
REPOSITORY_NAME={repo_id}
ALERT_NUMBER={alert_num}
REPOSITORY_KEY={repo_key}
ALERT_KEY={alert_key}
"""
STATE_ISSUE_SUMMARY = '[Code Scanning Issue States]'
STATE_ISSUE_KEY = util.make_key('gh2jira-state-issue')
STATE_ISSUE_TEMPLATE="""
This issue was automatically generated and contains states required for the synchronization between GitHub and JIRA.
DO NOT MODIFY DESCRIPTION BELOW LINE.
ISSUE_KEY={issue_key}
""".format(issue_key=STATE_ISSUE_KEY)
logger = logging.getLogger(__name__)
class Jira:
def __init__(self, url, user, token):
self.url = url
self.user = user
self.token = token
self.j = JIRA(url, basic_auth=(user, token))
def sync_tree_info(self, options=None):
if self._synced_tree_info:
Config.log(self.config_section, tag='FILE_DEST_META_ALREADY_SYNCED')
return
start = u.timestamp_now()
logging.info("starting FileDest metadata sync")
if options is None:
options = self._default_sync_options
# need to read persisted file, as that's the only place non-file-system metadata can live
self.read_tree_info()
# determine whether to force full refresh, only do it if tree has changed, on simply trust metadata:
do_full_refresh = False
if 'refresh_dest_meta' in options:
do_full_refresh = options['refresh_dest_meta']
# computing all those md5s takes a long time, so optionally skip it if the most
# recent modification time for _file_dest_root is unchanged since we last did it
if 'skip_refresh_if_tree_unchanged' in options:
last_mod = u.dir_last_modified(self._file_dest_root)
expected_last_mod = self._tree_last_modified
do_full_refresh = last_mod != expected_last_mod
msg = "last_mod do_full_refresh = '%s', last_mod = '%f', expected_last_mod = '%f'" % (
do_full_refresh, last_mod, expected_last_mod)
Config.log(msg, tag='FILE_DEST_META_TREE_UNCHANGED_TEST')
if do_full_refresh:
# physically walk the tree as it might not match persisted data
for dir_name, subdirs, files in os.walk(self._file_dest_root):
for file_name in files:
full_src = dir_name + '/' + file_name
setit = False
rel_path = u.make_rel_path(self._file_dest_root, dir_name, strict=False, no_leading_slash=True)
key = u.make_key(rel_path, file_name)
local_meta = u.local_metadata(dir_name, file_name)
local_meta['md5'] = u.md5(full_src)
if key in self.tree_info:
saved_meta = self.tree_info[key]
if 'md5' not in saved_meta:
saved_meta['md5'] = 'ERROR! md5 MISSING FROM tree_info!'
if local_meta['md5'] == saved_meta['md5']:
# sanity check
if local_meta['size'] != saved_meta['size']:
msg = "key '%s', saved: size %i, read: size %i" % (
key, saved_meta['size'], local_meta['size'])
Config.log(msg, tag='FILE_DEST_META_ERROR_NONFATAL')
# otherwise file is perfect, continue
else:
msg = "key '%s', md5 mismatch. saved: '%s', read: '%s'" % (
key, saved_meta['md5'], local_meta['md5'])
Config.log(msg, tag='FILE_DEST_META_ERROR_FATAL')
setit = True
else:
msg = "key '%s' not found in saved, adding" % key
Config.log(msg, tag='FILE_DEST_META_NEW_FILE')
setit = True
if setit:
local_meta['key'] = key
self.tree_info[key] = local_meta
# important: must never expose 'full' outside this class - it's a private
# implementation detail. Same for 'path'. Only 'key' is public
del local_meta['full']
del local_meta['path']
self.tree_info[key] = local_meta
self.tree_info[key]['_found_file_'] = True
missing = []
for key in self.tree_info:
if '_found_file_' in self.tree_info[key]:
del self.tree_info[key]['_found_file_']
else:
missing.append(key)
msg = "no file matching key '%s', deleting" % key
Config.log(msg, tag='FILE_DEST_META_MISSING')
for key in missing:
del self.tree_info[key]
self.write_tree_info()
act = "completed"
else:
# trust the persisted file (faster)
act = "bypassed"
elapsed = u.timestamp_now() - start
msg = "%s confirmation of tree info in %f seconds" % (act, elapsed)
Config.log(msg, tag='FILE_DEST_SYNC')
self._synced_tree_info = True
def upload_tree(self, local_root, remote_root='', options=None, local_tree_meta=None):
logging.info("starting FileDest upload")
if not options:
options = {'use_md5': True}
start = u.timestamp_now()
self._upload_count = 0
# refresh and save data for files already on dest
self.sync_tree_info(options=self._default_sync_options)
for dir_name, subdirs, files in os.walk(local_root):
rel_path = u.make_rel_path(local_root, dir_name)
if not rel_path.startswith('/tmp'):
for file_name in files:
local_file = dir_name + '/' + file_name
key = u.make_key(rel_path, file_name)
local_md5 = u.md5(local_file)
local_meta = None
if local_tree_meta and local_file in local_tree_meta:
local_meta = local_tree_meta[local_file]
else:
local_meta = u.local_metadata(dir_name, file_name)
size = local_meta['size']
cached_info = None
if key in self.tree_info:
cached_info = self.tree_info[key]
do_upload = True
if 'use_md5' in options:
if cached_info and not self.is_pending(key):
if 'md5' in self.tree_info[key]:
remote_md5 = self.tree_info[key]['md5']
do_upload = do_upload and remote_md5 != local_md5
else:
err = "no md5 value for existing key '%s' (old version?)" % key
logging.error(err)
else:
Config.log("file '%s' is not in FileDest" % key, tag='DEST_NO_EXISTING_FILE')
if self._max_upload_size >= 0 and size > self._max_upload_size:
logging.debug("file '%s' size (%i) > limit (%i), won't upload" %
(key, size, self._max_upload_size))
do_upload = False
if do_upload:
extra_args = {
'Metadata': {'md5': local_md5}
}
logging.debug("FileDest object upload starting, key = '%s', %i bytes" %
(key, size))
start = u.timestamp_now()
self._upload(dir_name, file_name, key, extra_args=extra_args)
rate = size / (u.timestamp_now() - start)
Config.log("key = '%s', %f bytes/sec" % (key, rate), tag='FILE_DEST_UPLOAD_OK')
# add metadata to our repos
info = {
'new': True,
'name': file_name,
'rel_path': rel_path,
'key': key,
'size': local_meta['size'],
'modified': local_meta['modified'],
# 'mod_dt': last_mod,
# 'e_tag': obj.e_tag,
'md5': local_md5
}
# transfer meta (e.g. thumbnail info) if exists
if local_tree_meta and local_file in local_tree_meta:
self.transfer_metadata(local_tree_meta[local_file], local_root=self.local_root, dest=info)
self.tree_info[key] = info
self._upload_count += 1
else:
Config.log("key = '%s'" % key, tag='FILE_DEST_UPLOAD_NO_CHANGE')
self.write_tree_info()
elapsed = u.timestamp_now() - start
logging.info("FileDest.upload_tree finished in %f seconds, uploaded %i files" %
(elapsed, self._upload_count))
return self._upload_count
