def transfer_metadata(self, finfo, local_root, dest): # TODO this can't be hard-coded for thumbs! generalize # transfer thumbnail info if exists if 'parent_full' in finfo: parent_full_path, parent_name = os.path.split(finfo['parent_full']) rel_path_parent = u.make_rel_path(local_root, parent_full_path, strict=True) dest['parent_key'] = u.make_key(rel_path_parent, parent_name) if 'thumb_full' in finfo: thumb_full_path, thumb_name = os.path.split(finfo['thumb_full']) rel_path_thumb = u.make_rel_path(local_root, thumb_full_path, strict=True) dest['thumb_key'] = u.make_key(rel_path_thumb, thumb_name) # copy other useful metadata if 'width' in finfo and 'height' in finfo: dest['width'] = finfo['width'] dest['height'] = finfo['height']
def parse_alert_info(desc): ''' Parse all the fields in an issue's description and return them as a tuple. If parsing fails for one of the fields, return a tuple of None's. ''' failed = None, None, None, None m = re.search('REPOSITORY_NAME=(.*)$', desc, re.MULTILINE) if m is None: return failed repo_id = m.group(1) m = re.search('ALERT_NUMBER=(.*)$', desc, re.MULTILINE) if m is None: return failed alert_num = int(m.group(1)) m = re.search('REPOSITORY_KEY=(.*)$', desc, re.MULTILINE) if m is None: return failed repo_key = m.group(1) m = re.search('ALERT_KEY=(.*)$', desc, re.MULTILINE) if m is None: return failed alert_key = m.group(1) # consistency checks: if repo_key != util.make_key(repo_id) \ or alert_key != util.make_alert_key(repo_id, alert_num): return failed return repo_id, alert_num, repo_key, alert_key
def create_issue(self, repo_id, rule_id, rule_desc, alert_url, alert_num): raw = self.j.create_issue( project=self.projectkey, summary='{prefix} {rule} in {repo}'.format( prefix=TITLE_PREFIX, rule=rule_id, repo=repo_id ), description=DESC_TEMPLATE.format( rule_desc=rule_desc, alert_url=alert_url, repo_id=repo_id, alert_num=alert_num, repo_key=util.make_key(repo_id), alert_key=util.make_alert_key(repo_id, alert_num) ), issuetype={'name': 'Bug'} ) logger.info('Created issue {issue_key} for alert {alert_num} in {repo_id}.'.format( issue_key=raw.key, alert_num=alert_num, repo_id=repo_id )) return JiraIssue(self, raw)
def fetch_issues(self, repo_id, alert_num=None): if alert_num is None: key = util.make_key(repo_id) else: key = util.make_alert_key(repo_id, alert_num) issue_search = 'project={jira_project} and description ~ "{key}"'.format( jira_project='\"{}\"'.format(self.projectkey), key=key ) issues = list(filter(lambda i: i.is_managed(), [JiraIssue(self, raw) for raw in self.j.search_issues(issue_search, maxResults=0)])) logger.debug('Search {search} returned {num_results} results.'.format( search=issue_search, num_results=len(issues) )) return issues
def get_tree_info(self, bucket=None, remote_root=''): self.tree_info.clear() for dir_name, subdirs, files in os.walk(self._file_dest_root): for file_name in files: full_src = dir_name + '/' + file_name rel_path = u.make_rel_path(self._file_dest_root, dir_name, strict=False, no_leading_slash=True) local_meta = u.local_metadata(dir_name, file_name) local_meta['key'] = u.make_key(rel_path, file_name) local_meta['md5'] = u.md5(full_src) # important: must never expose 'full' outside this class - it's a private # implementation detail. Same for 'path'. Only 'key' is public del local_meta['full'] del local_meta['path'] self.tree_info[local_meta['key']] = local_meta
def default_template_file_action(self, dir_name, file_name, dest_rel_path=None, dest_name=None): template_full = dir_name + '/' + file_name Config.log("default_template_file_action '%s'" % template_full, tag='DEFAULT_TEMPLATE_FILE_ACTION') if dest_name: rel_path = dest_rel_path dest_path = u.pathify(self.output_root, dest_rel_path) else: rel_path = u.make_rel_path(self.site_root, dir_name) dest_path = u.pathify(self.output_root, rel_path) dest_name = file_name u.ensure_path(dest_path) dest_full = u.pathify(dest_path, dest_name) info = { 'name': dest_name, 'path': dest_path, 'rel_path': rel_path, 'full': dest_full, 'key': u.make_key(rel_path, dest_name) } if self.config.is_template_type(file_name): template = open(template_full).read() output = u.debracket(template, self.interpret) if not self.config.is_special_file(info['key']): open(dest_full, 'w').write(output) local = u.local_metadata(dest_path, dest_name) info['size'] = local['size'] info['modified'] = local['modified'] info['md5'] = u.md5(dest_full) self.track_file(info) else: shutil.copyfile(template_full, dest_full) local = u.local_metadata(dest_path, dest_name) info['size'] = local['size'] info['modified'] = local['modified'] info['md5'] = u.md5(dest_full) self.track_file(info)
def track_file(self, finfo): full = finfo['full'] Config.log(full, tag='TP_TRACK_FILE') if 'md5' not in finfo: finfo['md5'] = u.md5(finfo['full']) if full in self.file_info: # don't replace finfo unless file has actually changed old_finfo = self.file_info[full] if finfo['md5'] == old_finfo['md5']: Config.log(full, tag='TP_TRACK_FILE_UNCHANGED') return self.file_info[full] = finfo if self._track_file_callback and self.will_upload(finfo['full']): if 'rel_path' not in finfo: finfo['rel_path'] = u.make_rel_path(self.config.output, finfo['path'], no_leading_slash=True) if 'key' not in finfo: finfo['key'] = u.make_key(finfo['rel_path'], finfo['name']) if ('size' not in finfo) or ('modified' not in finfo): logging.error('track_file (%s): finfo missing size and/or modified' % finfo['full']) tmp = u.local_metadata(finfo['path'], finfo['name']) finfo['size'] = tmp['size'] finfo['modified'] = tmp['modified'] self._track_file_callback(finfo)
{rule_desc} {alert_url} ---- This issue was automatically generated from a GitHub alert, and will be automatically resolved once the underlying problem is fixed. DO NOT MODIFY DESCRIPTION BELOW LINE. REPOSITORY_NAME={repo_id} ALERT_NUMBER={alert_num} REPOSITORY_KEY={repo_key} ALERT_KEY={alert_key} """ STATE_ISSUE_SUMMARY = '[Code Scanning Issue States]' STATE_ISSUE_KEY = util.make_key('gh2jira-state-issue') STATE_ISSUE_TEMPLATE=""" This issue was automatically generated and contains states required for the synchronization between GitHub and JIRA. DO NOT MODIFY DESCRIPTION BELOW LINE. ISSUE_KEY={issue_key} """.format(issue_key=STATE_ISSUE_KEY) logger = logging.getLogger(__name__) class Jira: def __init__(self, url, user, token): self.url = url self.user = user self.token = token self.j = JIRA(url, basic_auth=(user, token))
def sync_tree_info(self, options=None): if self._synced_tree_info: Config.log(self.config_section, tag='FILE_DEST_META_ALREADY_SYNCED') return start = u.timestamp_now() logging.info("starting FileDest metadata sync") if options is None: options = self._default_sync_options # need to read persisted file, as that's the only place non-file-system metadata can live self.read_tree_info() # determine whether to force full refresh, only do it if tree has changed, on simply trust metadata: do_full_refresh = False if 'refresh_dest_meta' in options: do_full_refresh = options['refresh_dest_meta'] # computing all those md5s takes a long time, so optionally skip it if the most # recent modification time for _file_dest_root is unchanged since we last did it if 'skip_refresh_if_tree_unchanged' in options: last_mod = u.dir_last_modified(self._file_dest_root) expected_last_mod = self._tree_last_modified do_full_refresh = last_mod != expected_last_mod msg = "last_mod do_full_refresh = '%s', last_mod = '%f', expected_last_mod = '%f'" % ( do_full_refresh, last_mod, expected_last_mod) Config.log(msg, tag='FILE_DEST_META_TREE_UNCHANGED_TEST') if do_full_refresh: # physically walk the tree as it might not match persisted data for dir_name, subdirs, files in os.walk(self._file_dest_root): for file_name in files: full_src = dir_name + '/' + file_name setit = False rel_path = u.make_rel_path(self._file_dest_root, dir_name, strict=False, no_leading_slash=True) key = u.make_key(rel_path, file_name) local_meta = u.local_metadata(dir_name, file_name) local_meta['md5'] = u.md5(full_src) if key in self.tree_info: saved_meta = self.tree_info[key] if 'md5' not in saved_meta: saved_meta['md5'] = 'ERROR! md5 MISSING FROM tree_info!' if local_meta['md5'] == saved_meta['md5']: # sanity check if local_meta['size'] != saved_meta['size']: msg = "key '%s', saved: size %i, read: size %i" % ( key, saved_meta['size'], local_meta['size']) Config.log(msg, tag='FILE_DEST_META_ERROR_NONFATAL') # otherwise file is perfect, continue else: msg = "key '%s', md5 mismatch. saved: '%s', read: '%s'" % ( key, saved_meta['md5'], local_meta['md5']) Config.log(msg, tag='FILE_DEST_META_ERROR_FATAL') setit = True else: msg = "key '%s' not found in saved, adding" % key Config.log(msg, tag='FILE_DEST_META_NEW_FILE') setit = True if setit: local_meta['key'] = key self.tree_info[key] = local_meta # important: must never expose 'full' outside this class - it's a private # implementation detail. Same for 'path'. Only 'key' is public del local_meta['full'] del local_meta['path'] self.tree_info[key] = local_meta self.tree_info[key]['_found_file_'] = True missing = [] for key in self.tree_info: if '_found_file_' in self.tree_info[key]: del self.tree_info[key]['_found_file_'] else: missing.append(key) msg = "no file matching key '%s', deleting" % key Config.log(msg, tag='FILE_DEST_META_MISSING') for key in missing: del self.tree_info[key] self.write_tree_info() act = "completed" else: # trust the persisted file (faster) act = "bypassed" elapsed = u.timestamp_now() - start msg = "%s confirmation of tree info in %f seconds" % (act, elapsed) Config.log(msg, tag='FILE_DEST_SYNC') self._synced_tree_info = True
def upload_tree(self, local_root, remote_root='', options=None, local_tree_meta=None): logging.info("starting FileDest upload") if not options: options = {'use_md5': True} start = u.timestamp_now() self._upload_count = 0 # refresh and save data for files already on dest self.sync_tree_info(options=self._default_sync_options) for dir_name, subdirs, files in os.walk(local_root): rel_path = u.make_rel_path(local_root, dir_name) if not rel_path.startswith('/tmp'): for file_name in files: local_file = dir_name + '/' + file_name key = u.make_key(rel_path, file_name) local_md5 = u.md5(local_file) local_meta = None if local_tree_meta and local_file in local_tree_meta: local_meta = local_tree_meta[local_file] else: local_meta = u.local_metadata(dir_name, file_name) size = local_meta['size'] cached_info = None if key in self.tree_info: cached_info = self.tree_info[key] do_upload = True if 'use_md5' in options: if cached_info and not self.is_pending(key): if 'md5' in self.tree_info[key]: remote_md5 = self.tree_info[key]['md5'] do_upload = do_upload and remote_md5 != local_md5 else: err = "no md5 value for existing key '%s' (old version?)" % key logging.error(err) else: Config.log("file '%s' is not in FileDest" % key, tag='DEST_NO_EXISTING_FILE') if self._max_upload_size >= 0 and size > self._max_upload_size: logging.debug("file '%s' size (%i) > limit (%i), won't upload" % (key, size, self._max_upload_size)) do_upload = False if do_upload: extra_args = { 'Metadata': {'md5': local_md5} } logging.debug("FileDest object upload starting, key = '%s', %i bytes" % (key, size)) start = u.timestamp_now() self._upload(dir_name, file_name, key, extra_args=extra_args) rate = size / (u.timestamp_now() - start) Config.log("key = '%s', %f bytes/sec" % (key, rate), tag='FILE_DEST_UPLOAD_OK') # add metadata to our repos info = { 'new': True, 'name': file_name, 'rel_path': rel_path, 'key': key, 'size': local_meta['size'], 'modified': local_meta['modified'], # 'mod_dt': last_mod, # 'e_tag': obj.e_tag, 'md5': local_md5 } # transfer meta (e.g. thumbnail info) if exists if local_tree_meta and local_file in local_tree_meta: self.transfer_metadata(local_tree_meta[local_file], local_root=self.local_root, dest=info) self.tree_info[key] = info self._upload_count += 1 else: Config.log("key = '%s'" % key, tag='FILE_DEST_UPLOAD_NO_CHANGE') self.write_tree_info() elapsed = u.timestamp_now() - start logging.info("FileDest.upload_tree finished in %f seconds, uploaded %i files" % (elapsed, self._upload_count)) return self._upload_count