def scrapeNightlies(self, connection, product_name, date):
nightly_url = urljoin(self.config.base_url, product_name, 'nightly',
date.strftime('%Y'), date.strftime('%m'), '')
cursor = connection.cursor()
dir_prefix = date.strftime('%Y-%m-%d')
nightlies = getLinks(nightly_url, startswith=dir_prefix)
for nightly in nightlies:
for info in getNightly(nightly, nightly_url):
platform, repository, version, kvpairs, bad_lines = info
for bad_line in bad_lines:
self.config.logger.warning("Bad line for %s (%r)", nightly,
bad_line)
build_type = 'Nightly'
if version.endswith('a2'):
build_type = 'Aurora'
if kvpairs.get('buildID'):
build_id = kvpairs['buildID']
buildutil.insert_build(cursor,
product_name,
version,
platform,
build_id,
build_type,
None,
repository,
ignore_duplicates=True)
def scrapeB2G(config, cursor, product_name, urllib=urllib2, date=None):
month = date.strftime('%m')
b2g_url = '%s/%s/%s/' % (config.base_url, product_name,
'manifests')
try:
day = date.strftime('%d')
dir_prefix = '%s-%s-%s' % (date.year, month, day)
# I have no idea what this first level of directories means :/
# TODO get info about that and update this search
version_dirs = getLinks(b2g_url, startswith='1.', urllib=urllib)
for version_dir in version_dirs:
# /1.0.0/2013/01/2013-01-27-07/*.json
prod_url = '%s/%s/%s/%s/' % (b2g_url, version_dir, date.year, month)
nightlies = getLinks(prod_url, startswith=dir_prefix, urllib=urllib)
for nightly in nightlies:
for info in getB2G(nightly, prod_url):
(platform, repository, version, kvpairs) = info
build_id = kvpairs['buildid']
build_type = kvpairs['build_type']
buildutil.insert_build(cursor, product_name, version, platform,
build_id, build_type, kvpairs.get('beta_number', None), repository,
ignore_duplicates=True)
except urllib.URLError:
util.reportExceptionAndContinue(logger)
def scrapeReleases(config, cursor, product_name, urllib=urllib2):
prod_url = '%s/%s/' % (config.base_url, product_name)
# releases are sometimes in nightly, sometimes in candidates dir.
# look in both.
for directory in ('nightly', 'candidates'):
if not getLinks(prod_url, startswith=directory, urllib=urllib):
logger.debug('Dir %s not found for %s' % (directory, product_name))
continue
url = '%s/%s/%s/' % (config.base_url, product_name, directory)
try:
releases = getLinks(url, endswith='-candidates/',
urllib=urllib)
for release in releases:
for info in getRelease(release, url):
(platform, version, build_number, kvpairs) = info
build_type = 'Release'
beta_number = None
repository = 'mozilla-release'
if 'b' in version:
build_type = 'Beta'
version, beta_number = version.split('b')
repository = 'mozilla-beta'
build_id = kvpairs['buildID']
buildutil.insert_build(cursor, product_name, version,
platform, build_id, build_type,
beta_number, repository)
except urllib.URLError:
util.reportExceptionAndContinue(logger)
def _insert_build(self, cursor, *args, **kwargs):
if self.config.dry_run:
print "INSERT BUILD"
print args
print kwargs
else:
buildutil.insert_build(cursor, *args, **kwargs)
def create(self, **kwargs):
"""
Create a new build for a product.
See http://socorro.readthedocs.org/en/latest/middleware.html#builds
Required keyword arguments:
product - Concerned product, e.g. firefox
version - Concerned version, e.g. 9.0a1
platform - Platform for this build, e.g. win32
build_id - Build ID for this build (yyyymmdd######)
build_type - Type of build, e.g. Nightly, Beta, Aurora, Release
Required if build_type is Beta:
beta_number - The beta number, e.g. 9.0b#
Optional keyword arguments:
repository - Repository this build came from
Return: (product_name, version)
"""
# Parse arguments
filters = [
("product", None, "str"),
("version", None, "str"),
("platform", None, "str"),
("build_id", None, "str"),
("build_type", None, "str"),
("beta_number", None, "str"),
("repository", "", "str")
]
params = external_common.parse_arguments(filters, kwargs)
self._require_parameters(params, "product", "version", "platform",
"build_id", "build_type")
if params["build_type"].lower() == "beta":
self._require_parameters(params, "beta_number")
try:
connection = self.database.connection()
cursor = connection.cursor()
buildutil.insert_build(cursor, params["product"],
params["version"], params["platform"],
params["build_id"], params["build_type"],
params["beta_number"],
params["repository"])
except psycopg2.Error:
logger.error("Failed inserting build data into PostgresSQL",
exc_info=True)
connection.rollback()
raise
else:
connection.commit()
finally:
connection.close()
return (params["product"], params["version"])
def scrapeNightlies(config, cursor, product_name, urllib=urllib2, date=None):
month = date.strftime('%m')
nightly_url = '%s/%s/%s/%s/%s/' % (config.base_url, product_name,
'nightly', date.year, month)
try:
day = date.strftime('%d')
dir_prefix = '%s-%s-%s' % (date.year, month, day)
nightlies = getLinks(nightly_url, startswith=dir_prefix, urllib=urllib)
for nightly in nightlies:
for info in getNightly(nightly, nightly_url):
(platform, repository, version, kvpairs) = info
build_id = kvpairs['buildID']
build_type = 'Nightly'
if version.endswith('a2'):
build_type = 'Aurora'
buildutil.insert_build(cursor,
product_name,
version,
platform,
build_id,
build_type,
None,
repository,
ignore_duplicates=True)
except urllib.URLError:
util.reportExceptionAndContinue(logger)
def create(self, **kwargs):
"""
Create a new build for a product.
See http://socorro.readthedocs.org/en/latest/middleware.html#builds
Required keyword arguments:
product - Concerned product, e.g. firefox
version - Concerned version, e.g. 9.0a1
platform - Platform for this build, e.g. win32
build_id - Build ID for this build (yyyymmdd######)
build_type - Type of build, e.g. Nightly, Beta, Aurora, Release
Required if build_type is Beta:
beta_number - The beta number, e.g. 9.0b#
Optional keyword arguments:
repository - Repository this build came from
Return: (product_name, version)
"""
# Parse arguments
filters = [
("product", None, "str"),
("version", None, "str"),
("platform", None, "str"),
("build_id", None, "int"),
("build_type", None, "str"),
("beta_number", None, "int"),
("repository", "", "str")
]
params = external_common.parse_arguments(filters, kwargs)
self._require_parameters(params, "product", "version", "platform",
"build_id", "build_type")
if params["build_type"].lower() == "beta":
self._require_parameters(params, "beta_number")
connection = None
try:
connection = self.database.connection()
cursor = connection.cursor()
buildutil.insert_build(cursor, params["product"],
params["version"], params["platform"],
params["build_id"], params["build_type"],
params["beta_number"],
params["repository"])
except psycopg2.Error, e:
error = str(e)
logger.error("Failed inserting build data into PostgresSQL, "
"reason: %s" % error,
exc_info=True)
connection.rollback()
if "CONTEXT" in error:
error = error[0:error.index("CONTEXT")]
raise InsertionError(error)
def scrapeB2G(self, connection, product_name, date):
if not product_name == 'b2g':
return
cursor = connection.cursor()
b2g_manifests = urljoin(self.config.base_url, product_name,
'manifests')
dir_prefix = date.strftime('%Y-%m-%d')
version_dirs = getLinks(b2g_manifests, startswith='1.')
for version_dir in version_dirs:
prod_url = urljoin(b2g_manifests, version_dir,
date.strftime('%Y'), date.strftime('%m'))
nightlies = getLinks(prod_url, startswith=dir_prefix)
for nightly in nightlies:
for info in getB2G(nightly, prod_url, backfill_date=None, logger=self.config.logger):
(platform, repository, version, kvpairs) = info
build_id = kvpairs['buildid']
build_type = kvpairs['build_type']
buildutil.insert_build(
cursor,
product_name,
version,
platform,
build_id,
build_type,
kvpairs.get('beta_number', None),
repository,
ignore_duplicates=True
)
def scrapeReleases(config, cursor, product_name, urllib=urllib2):
prod_url = "%s/%s/" % (config.base_url, product_name)
# releases are sometimes in nightly, sometimes in candidates dir.
# look in both.
for directory in ("nightly", "candidates"):
if not getLinks(prod_url, startswith=directory, urllib=urllib):
logger.debug("Dir %s not found for %s" % (directory, product_name))
continue
url = "%s/%s/%s/" % (config.base_url, product_name, directory)
try:
releases = getLinks(url, endswith="-candidates/", urllib=urllib)
for release in releases:
for info in getRelease(release, url):
(platform, version, build_number, kvpairs) = info
build_type = "Release"
beta_number = None
repository = "mozilla-release"
if "b" in version:
build_type = "Beta"
version, beta_number = version.split("b")
repository = "mozilla-beta"
build_id = kvpairs["buildID"]
buildutil.insert_build(
cursor, product_name, version, platform, build_id, build_type, beta_number, repository
)
except urllib.URLError:
util.reportExceptionAndContinue(logger)
def scrapeReleases(self, connection, product_name):
prod_url = urljoin(self.config.base_url, product_name, '')
# releases are sometimes in nightly, sometimes in candidates dir.
# look in both.
logger = self.config.logger
cursor = connection.cursor()
for directory in ('nightly', 'candidates'):
if not getLinks(prod_url, startswith=directory):
logger.debug('Dir %s not found for %s',
directory, product_name)
continue
url = urljoin(self.config.base_url, product_name, directory, '')
releases = getLinks(url, endswith='-candidates/')
for release in releases:
for info in getRelease(release, url):
platform, version, build_number, kvpairs = info
build_type = 'Release'
beta_number = None
repository = 'mozilla-release'
if 'b' in version:
build_type = 'Beta'
version, beta_number = version.split('b')
repository = 'mozilla-beta'
build_id = kvpairs['buildID']
buildutil.insert_build(cursor,
product_name,
version,
platform,
build_id,
build_type,
beta_number,
repository,
ignore_duplicates=True)
def _insert_build(self, cursor, *args, **kwargs):
if self.config.dry_run:
print "INSERT BUILD"
print args
print kwargs
else:
buildutil.insert_build(cursor, *args, **kwargs)
def scrapeB2G(self, connection, product_name, date):
if not product_name == 'b2g':
return
cursor = connection.cursor()
b2g_manifests = urljoin(self.config.base_url, product_name,
'manifests', 'nightly')
dir_prefix = date.strftime('%Y-%m-%d')
version_dirs = getLinks(b2g_manifests, startswith='1.')
for version_dir in version_dirs:
prod_url = urljoin(b2g_manifests, version_dir, date.strftime('%Y'),
date.strftime('%m'))
nightlies = getLinks(prod_url, startswith=dir_prefix)
for nightly in nightlies:
b2gs = getB2G(nightly,
prod_url,
backfill_date=None,
logger=self.config.logger)
for info in b2gs:
(platform, repository, version, kvpairs) = info
build_id = kvpairs['buildid']
build_type = kvpairs['build_type']
buildutil.insert_build(cursor,
product_name,
version,
platform,
build_id,
build_type,
kvpairs.get('beta_number', None),
repository,
ignore_duplicates=True)
def scrapeNightlies(self, connection, product_name, date):
nightly_url = urljoin(self.config.base_url, product_name, 'nightly',
date.strftime('%Y'),
date.strftime('%m'),
'')
cursor = connection.cursor()
dir_prefix = date.strftime('%Y-%m-%d')
nightlies = getLinks(nightly_url, startswith=dir_prefix)
for nightly in nightlies:
for info in getNightly(nightly, nightly_url):
platform, repository, version, kvpairs, bad_lines = info
for bad_line in bad_lines:
self.config.logger.warning(
"Bad line for %s (%r)",
nightly, bad_line
)
build_type = 'Nightly'
if version.endswith('a2'):
build_type = 'Aurora'
if kvpairs.get('buildID'):
build_id = kvpairs['buildID']
buildutil.insert_build(
cursor,
product_name,
version,
platform,
build_id,
build_type,
None,
repository,
ignore_duplicates=True
)
def _insert_build(self, cursor, *args, **kwargs):
if self.config.dry_run:
print "INSERT BUILD"
for arg in args:
print "\t", repr(arg)
for key in kwargs:
print "\t%s=" % key, repr(kwargs[key])
else:
buildutil.insert_build(cursor, *args, **kwargs)
def _insert_build(self, cursor, *args, **kwargs):
if self.config.dry_run:
print "INSERT BUILD"
for arg in args:
print "\t", repr(arg)
for key in kwargs:
print "\t%s=" % key, repr(kwargs[key])
else:
buildutil.insert_build(cursor, *args, **kwargs)
def test_insert_build(self):
cursor = self.connection.cursor()
# Test 1: successfully insert a build
buildutil.insert_build(cursor, 'Firefox', 'VERSIONAME5',
'PLATFORMNAME5', '20110101', 'Release', '5', 'REPO5')
actual = self.build_exists(cursor, 'Firefox',
'VERSIONAME5', 'PLATFORMNAME5', '20110101', 'Release',
'5', 'REPO5')
self.assertTrue(actual)
# Test 2: fail at inserting a build
buildutil.insert_build(cursor, 'Unknown', 'VERSIONAME5', 'PLATFORMNAME5',
'20110101', 'Release', '5', 'REPO5')
actual = self.build_exists(cursor, 'Unknown',
'VERSIONAME5', 'PLATFORMNAME5', '20110101', 'Release',
'5', 'REPO5')
self.assertFalse(actual)
def test_insert_build(self):
me.cur = me.conn.cursor(cursor_factory=psy.LoggingCursor)
me.cur.setLogger(me.fileLogger)
sql = """DELETE FROM releases_raw
WHERE product_name = 'PRODUCTNAME5'"""
me.cur.execute(sql)
me.cur.connection.commit()
try:
buildutil.insert_build(me.cur, 'PRODUCTNAME5', 'VERSIONAME5',
'PLATFORMNAME5', '5', 'BUILDTYPE5', '5', 'REPO5')
actual = buildutil.build_exists(me.cur, 'PRODUCTNAME5',
'VERSIONAME5', 'PLATFORMNAME5', '5', 'BUILDTYPE5',
'5', 'REPO5')
assert actual == 1, "expected 1, got %s" % (actual)
except Exception, x:
print "Exception in do_insert_build() ... Error: ", type(x), x
socorro.lib.util.reportExceptionAndAbort(me.fileLogger)
def test_insert_build(self):
cursor = self.connection.cursor()
# Test 1: successfully insert a build
buildutil.insert_build(cursor, 'Firefox', 'VERSIONAME5',
'PLATFORMNAME5', '20110101', 'Release', '5',
'REPO5')
actual = self.build_exists(cursor, 'Firefox', 'VERSIONAME5',
'PLATFORMNAME5', '20110101', 'Release', '5',
'REPO5')
self.assertTrue(actual)
# Test 2: fail at inserting a build
buildutil.insert_build(cursor, 'Unknown', 'VERSIONAME5',
'PLATFORMNAME5', '20110101', 'Release', '5',
'REPO5')
actual = self.build_exists(cursor, 'Unknown', 'VERSIONAME5',
'PLATFORMNAME5', '20110101', 'Release', '5',
'REPO5')
self.assertFalse(actual)
def scrapeNightlies(config, cursor, product_name, urllib=urllib2, date=None):
month = date.strftime("%m")
nightly_url = "%s/%s/%s/%s/%s/" % (config.base_url, product_name, "nightly", date.year, month)
try:
day = date.strftime("%d")
dir_prefix = "%s-%s-%s" % (date.year, month, day)
nightlies = getLinks(nightly_url, startswith=dir_prefix, urllib=urllib)
for nightly in nightlies:
for info in getNightly(nightly, nightly_url):
(platform, repository, version, kvpairs) = info
build_id = kvpairs["buildID"]
build_type = "Nightly"
if version.endswith("a2"):
build_type = "Aurora"
buildutil.insert_build(cursor, product_name, version, platform, build_id, build_type, None, repository)
except urllib.URLError:
util.reportExceptionAndContinue(logger)
def test_insert_build(self):
cursor = self.connection.cursor()
# Test 1: successfully insert a build
buildutil.insert_build(
cursor, "Firefox", "VERSIONAME5", "PLATFORMNAME5", "20110101", "Release", "5", "REPO5", "build1"
)
actual = self.build_exists(
cursor, "Firefox", "VERSIONAME5", "PLATFORMNAME5", "20110101", "Release", "5", "REPO5", "build1"
)
ok_(actual)
# Test 2: fail at inserting a build
buildutil.insert_build(
cursor, "Unknown", "VERSIONAME5", "PLATFORMNAME5", "20110101", "Release", "5", "REPO5", "build1"
)
actual = self.build_exists(
cursor, "Unknown", "VERSIONAME5", "PLATFORMNAME5", "20110101", "Release", "5", "REPO5", "build1"
)
ok_(not actual)
def scrapeNightlies(config, cursor, product_name, urllib=urllib2, date=None):
month = date.strftime('%m')
nightly_url = '%s/%s/%s/%s/%s/' % (config.base_url, product_name,
'nightly', date.year, month)
try:
day = date.strftime('%d')
dir_prefix = '%s-%s-%s' % (date.year, month, day)
nightlies = getLinks(nightly_url, startswith=dir_prefix,
urllib=urllib)
for nightly in nightlies:
for info in getNightly(nightly, nightly_url):
(platform, repository, version, kvpairs) = info
build_id = kvpairs['buildID']
build_type = 'Nightly'
if version.endswith('a2'):
build_type = 'Aurora'
buildutil.insert_build(cursor, product_name, version, platform,
build_id, build_type, None, repository)
except urllib.URLError:
util.reportExceptionAndContinue(logger)
def scrapeNightlies(self, connection, product_name, date):
nightly_url = urljoin(self.config.base_url, product_name, 'nightly',
date.strftime('%Y'),
date.strftime('%m'),
'')
cursor = connection.cursor()
dir_prefix = date.strftime('%Y-%m-%d')
nightlies = getLinks(nightly_url, startswith=dir_prefix)
for nightly in nightlies:
for info in getNightly(nightly, nightly_url):
platform, repository, version, kvpairs = info
build_id = kvpairs['buildID']
build_type = 'Nightly'
if version.endswith('a2'):
build_type = 'Aurora'
buildutil.insert_build(cursor,
product_name,
version,
platform,
build_id,
build_type,
None,
repository)
def scrapeReleases(config, cursor, product_name, urllib=urllib2):
prod_url = '%s/%s/' % (config.base_url, product_name)
# releases are sometimes in nightly, sometimes in candidates dir.
# look in both.
for directory in ('nightly', 'candidates'):
if not getLinks(prod_url, startswith=directory, urllib=urllib):
logger.debug('Dir %s not found for %s' % (directory, product_name))
continue
url = '%s/%s/%s/' % (config.base_url, product_name, directory)
try:
releases = getLinks(url, endswith='-candidates/', urllib=urllib)
for release in releases:
for info in getRelease(release, url):
(platform, version, build_number, kvpairs) = info
build_type = 'Release'
beta_number = None
repository = 'mozilla-release'
if 'b' in version:
build_type = 'Beta'
version, beta_number = version.split('b')
repository = 'mozilla-beta'
build_id = kvpairs['buildID']
buildutil.insert_build(cursor,
product_name,
version,
platform,
build_id,
build_type,
beta_number,
repository,
ignore_duplicates=True)
except urllib.URLError:
util.reportExceptionAndContinue(logger)
