def do(self):
crawler = Crawler(self.url)
crawler.run()
links = crawler.get_all_links()
git_urls = {}
# Try to get the git clone URL from the web viewer
git_clone_domain = re.match('.*?(.*?)gitweb.*?', self.url).group(1)
git_clone_url = '{0}{1}/'.format(git_clone_domain, 'git')
for link in links:
# Url have the format http://git.lxde.org/gitweb/?p=lxde/lxqt-config-randr.git;a=tree
match = re.match('.*?p=(.*?);.*?', link)
if match:
name = match.group(1)
git_urls[name] = git_clone_url + name
# The Gerrit project has a single fileset assigned (this)
# We empty the fileset and add dynamically the ones referenced by Gerrit
self.project.filesets = []
for project in git_urls.keys():
fileset_name = project
url = git_urls[project]
fileset = GitFileSet(self.project_name, fileset_name, url, '')
fileset.set_pattern('.*?ca.po')
logging.debug("Gitweb adding {0}-{1}".format(self.project_name, name))
self.project.add(fileset)
self.project.do()
def do(self):
# Download JSON file
download = DownloadFile()
download.get_file(self.url, self.filename)
self._remove_first_line_from_file(self.filename)
# The Gerrit project has a single fileset assigned (this)
# We empty the fileset and add dynamically the ones referenced by Gerrit
self.project.filesets = []
with open(self.filename) as json_data:
data = json.load(json_data)
# Get every project entry
for attribute, value in data.items():
name = None
url = None
for prj_attribute, prj_value in value.iteritems():
if prj_attribute == 'name':
name = prj_value
elif prj_attribute == 'clone_url':
url = prj_value
if not re.match(self.pattern, name):
logging.debug('GerritDirectoryFileSet. Discarding:' + name)
continue
fileset = GitFileSet(self.project_name, name, url, '')
# Some Android projects contain there own po files like
# https://android.googlesource.com/platform/ndk and they have
# the name standard "ca.po"
# The rest are produced by a2po then they have the pattern '-ca.po'
fileset.set_pattern('.*?ca.po')
logging.debug("Gerrit adding {0}-{1}".format(
self.project_name, name))
self.project.add(fileset)
# All the new filesets have been added re-process project now
logging.debug(
'GerritDirectoryFileSet. Added {0} filesets dynamically'.format(
len(self.project.filesets)))
self.project.do()
def do(self):
# Download JSON file
download = DownloadFile()
download.get_file(self.url, self.filename)
self._remove_first_line_from_file(self.filename)
# The Gerrit project has a single fileset assigned (this)
# We empty the fileset and add dynamically the ones referenced by Gerrit
self.project.filesets = []
with open(self.filename) as json_data:
data = json.load(json_data)
# Get every project entry
for attribute, value in data.items():
name = None
url = None
for prj_attribute, prj_value in value.iteritems():
if prj_attribute == 'name':
name = prj_value
elif prj_attribute == 'clone_url':
url = prj_value
if not re.match(self.pattern, name):
logging.debug('GerritDirectoryFileSet. Discarding:' + name)
continue
fileset = GitFileSet(self.project_name, name, url, '')
# Some Android projects contain there own po files like
# https://android.googlesource.com/platform/ndk and they have
# the name standard "ca.po"
# The rest are produced by a2po then they have the pattern '-ca.po'
fileset.set_pattern('.*?ca.po')
logging.debug("Gerrit adding {0}-{1}".format(self.project_name, name))
self.project.add(fileset)
# All the new filesets have been added re-process project now
logging.debug('GerritDirectoryFileSet. Added {0} filesets dynamically'.
format(len(self.project.filesets)))
self.project.do()
def add_filesets(self, project_dto):
for fileset in project_dto.filesets:
logging.debug(fileset)
if fileset.type == 'local-file':
fs = LocalFileSet(self.name, fileset.name, fileset.url,
fileset.target)
elif fileset.type == 'compressed':
fs = CompressedFileSet(self.name, fileset.name, fileset.url,
fileset.target)
fs.set_pattern(fileset.pattern)
elif fileset.type == 'bazaar':
fs = BazaarFileSet(self.name, fileset.name, fileset.url,
fileset.target)
fs.set_pattern(fileset.pattern)
elif fileset.type == 'transifex':
fs = TransifexFileSet(self.name, fileset.name, fileset.url,
fileset.target)
elif fileset.type == 'transifexhub':
fs = TransifexHubFileSet(self.name, fileset.name, fileset.url,
fileset.target)
fs.set_project(self)
elif fileset.type == 'local-dir':
fs = LocalDirFileSet(self.name, fileset.name, fileset.url,
fileset.target)
elif fileset.type == 'file':
fs = FileFileSet(self.name, fileset.name, fileset.url,
fileset.target)
elif fileset.type == 'subversion':
fs = SubversionFileSet(self.name, fileset.name, fileset.url,
fileset.target)
elif fileset.type == 'crawl':
fs = CrawlFileSet(self.name, fileset.name, fileset.url,
fileset.target)
fs.set_pattern(fileset.pattern)
elif fileset.type == 'git':
fs = GitFileSet(self.name, fileset.name, fileset.url,
fileset.target)
fs.set_pattern(fileset.pattern)
elif fileset.type == 'gerrit-directory':
fs = GerritDirectoryFileSet(self.name, fileset.name,
fileset.url, fileset.target)
fs.set_pattern(fileset.pattern)
fs.set_project(self)
else:
msg = 'Unsupported filetype: {0}'
logging.error(msg.format(fileset.type))
self.add(fs)
fs.add_excluded(fileset.excluded)