def xpi_build_from_model(rev_pk, mod_codes={}, att_codes={}, hashtag=None, tqueued=None): """ Get object and build xpi """ if not hashtag: log.critical("No hashtag provided") return tstart = time.time() if tqueued: tinqueue = (tstart - tqueued) * 1000 statsd.timing('xpi.build.queued', tinqueue) log.info('[xpi:%s] Addon job picked from queue (%dms)' % (hashtag, tinqueue)) revision = PackageRevision.objects.get(pk=rev_pk) log.debug('[xpi:%s] Building %s' % (hashtag, revision)) # prepare changed modules and attachments modules = [] attachments = [] for mod in revision.modules.all(): if str(mod.pk) in mod_codes: mod.code = mod_codes[str(mod.pk)] modules.append(mod) for att in revision.attachments.all(): if str(att.pk) in att_codes: att.code = att_codes[str(att.pk)] attachments.append(att) revision.build_xpi(modules=modules, attachments=attachments, hashtag=hashtag, tstart=tstart)
def check_redis(): redis = caching.invalidation.get_redis_backend() try: return redis.info(), None except Exception, e: log.critical('Failed to chat with redis: (%s)' % e) return None, e
def update_monolith_stats(metric, date, **kw): log.info('Updating monolith statistics (%s) for (%s)' % (metric, date)) jobs = _get_monolith_jobs(date)[metric] for job in jobs: try: # Only record if count is greater than zero. count = job['count']() if count: value = {'count': count} if 'dimensions' in job: value.update(job['dimensions']) MonolithRecord.objects.create(recorded=date, key=metric, value=json.dumps(value)) log.debug('Monolith stats details: (%s) has (%s) for (%s). ' 'Value: %s' % (metric, count, date, value)) else: log.debug('Monolith stat (%s) did not record due to falsy ' 'value (%s) for (%s)' % (metric, count, date)) except Exception as e: log.critical('Update of monolith table failed: (%s): %s' % ([metric, date], e))
def monitor(request): # For each check, a boolean pass/fail status to show in the template status_summary = {} status = 200 # Check all memcached servers scheme, servers, _ = parse_backend_uri(settings.CACHE_BACKEND) memcache_results = [] status_summary['memcache'] = True if 'memcached' in scheme: hosts = servers.split(';') for host in hosts: ip, port = host.split(':') try: s = socket.socket() s.connect((ip, int(port))) except Exception, e: result = False status_summary['memcache'] = False log.critical('Failed to connect to memcached (%s): %s' % (host, e)) else: result = True finally:
def xpi_build_from_model(rev_pk, mod_codes={}, att_codes={}, hashtag=None, tqueued=None): """ Get object and build xpi """ if not hashtag: log.critical("No hashtag provided") return tstart = time.time() if tqueued: tinqueue = (tstart - tqueued) * 1000 statsd.timing('xpi.build.queued', tinqueue) log.info('[xpi:%s] Addon job picked from queue (%dms)' % (hashtag, tinqueue)) revision = PackageRevision.objects.get(pk=rev_pk) log.debug('[xpi:%s] Building %s' % (hashtag, revision)) # prepare changed modules and attachments modules = [] attachments = [] for mod in revision.modules.all(): if str(mod.pk) in mod_codes: mod.code = mod_codes[str(mod.pk)] modules.append(mod) for att in revision.attachments.all(): if str(att.pk) in att_codes: att.code = att_codes[str(att.pk)] attachments.append(att) revision.build_xpi( modules=modules, attachments=attachments, hashtag=hashtag, tstart=tstart)
def monitor(request): status = True data = {} # Check Read/Write filepaths = [ (settings.UPLOAD_DIR, os.R_OK | os.W_OK, 'We want read + write.'), ] if hasattr(settings, 'XPI_TARGETDIR'): filepaths.append((settings.XPI_TARGETDIR, os.R_OK | os.W_OK, 'We want read + write. Should be a shared directory ' 'on multiserver installations')) for sdk in SDK.objects.all(): filepaths.append((sdk.get_source_dir(), os.R_OK, 'We want read on %s' % sdk.version),) filepath_results = [] filepath_status = True for path, perms, notes in filepaths: path_exists = os.path.isdir(path) path_perms = os.access(path, perms) filepath_status = filepath_status and path_exists and path_perms if not filepath_status and status: status = False filepath_results.append((path, path_exists, path_perms, notes)) # free space on XPI_TARGETDIR disk x_path = '%s/' % settings.XPI_TARGETDIR s_path = '%s/' % settings.SDKDIR_PREFIX x = os.statvfs(x_path) s = os.statvfs(s_path) data['free'] = { 'xpi_targetdir %s' % x_path: (x.f_bavail * x.f_frsize) / 1024, 'sdkdir_prefix %s' % s_path: (s.f_bavail * s.f_frsize) / 1024 } data['filepaths'] = filepath_results # Check celery try: data['celery_responses'] = CeleryResponse.objects.all() except: status = False # Check ElasticSearch try: es = get_es() data['es_health'] = es.cluster_health() data['es_health']['version'] = es.collect_info()['server']['version']['number'] if data['es_health']['status'] =='red': status = False log.warning('ElasticSearch cluster health was red.') except Exception, e: status = False log.critical('Failed to connect to ElasticSearch: %s' % e)
def monitor(request): status = True data = {} # Check Read/Write filepaths = [ (settings.UPLOAD_DIR, os.R_OK | os.W_OK, 'We want read + write.'), ] if hasattr(settings, 'XPI_TARGETDIR'): filepaths.append((settings.XPI_TARGETDIR, os.R_OK | os.W_OK, 'We want read + write. Should be a shared directory ' 'on multiserver installations')) for sdk in SDK.objects.all(): filepaths.append((sdk.get_source_dir(), os.R_OK, 'We want read on %s' % sdk.version), ) filepath_results = [] filepath_status = True for path, perms, notes in filepaths: path_exists = os.path.isdir(path) path_perms = os.access(path, perms) filepath_status = filepath_status and path_exists and path_perms if not filepath_status and status: status = False filepath_results.append((path, path_exists, path_perms, notes)) # free space on XPI_TARGETDIR disk x_path = '%s/' % settings.XPI_TARGETDIR s_path = '%s/' % settings.SDKDIR_PREFIX x = os.statvfs(x_path) s = os.statvfs(s_path) data['free'] = [('xpi_targetdir %s' % x_path, x.f_bavail * x.f_frsize), ('sdkdir_prefix %s' % s_path, s.f_bavail * s.f_frsize)] data['filepaths'] = filepath_results # Check celery try: data['celery_responses'] = CeleryResponse.objects.all() except: status = False # Check ElasticSearch try: es = get_es() data['es_health'] = es.cluster_health() data['es_health']['version'] = es.collect_info( )['server']['version']['number'] if data['es_health']['status'] == 'red': status = False log.warning('ElasticSearch cluster health was red.') except Exception, e: status = False log.critical('Failed to connect to ElasticSearch: %s' % e)
def zip_source(pk, hashtag, tqueued=None, **kw): if not hashtag: log.critical("[zip] No hashtag provided") return tstart = time.time() if tqueued: tinqueue = (tstart - tqueued) * 1000 statsd.timing('zip.queued', tinqueue) log.info('[zip:%s] Addon job picked from queue (%dms)' % (hashtag, tinqueue)) log.debug("[zip:%s] Compressing" % pk) PackageRevision.objects.get(pk=pk).zip_source(hashtag=hashtag, tstart=tstart) log.debug("[zip:%s] Compressed" % pk)
def get_amo_cursor(): import MySQLdb try: auth_conn = MySQLdb.connect( host=settings.AUTH_DATABASE['HOST'], user=settings.AUTH_DATABASE['USER'], passwd=settings.AUTH_DATABASE['PASSWORD'], db=settings.AUTH_DATABASE['NAME']) except Exception, err: log.critical("Authentication database connection failure: %s" % str(err)) raise
def build(sdk_dir, package_dir, filename, hashtag, tstart=None, options=None, temp_dir=None): """Build xpi from SDK with prepared packages in sdk_dir. :params: * sdk_dir (String) SDK directory * package_dir (string) dir of the Add-on package * filename (string) XPI will be build with this name * hashtag (string) XPI will be copied to a file which name is creted using the unique hashtag * t1 (integer) time.time() of the process started :returns: (list) ``cfx xpi`` response where ``[0]`` is ``stdout`` and ``[1]`` ``stderr`` """ t1 = time.time() if not temp_dir: temp_dir = sdk_dir # create XPI os.chdir(package_dir) cfx = [ settings.PYTHON_EXEC, "%s/bin/cfx" % sdk_dir, "--binary=%s" % settings.XULRUNNER_BINARY, "--keydir=%s/%s" % (sdk_dir, settings.KEYDIR), "xpi", ] if options: cfx.append(options) log.debug(cfx) info_targetfilename = "%s.json" % hashtag info_targetpath = os.path.join(settings.XPI_TARGETDIR, info_targetfilename) env = dict( PATH="%s/bin:%s" % (sdk_dir, os.environ["PATH"]), VIRTUAL_ENV=sdk_dir, CUDDLEFISH_ROOT=sdk_dir, PYTHONPATH=os.path.join(sdk_dir, "python-lib"), ) try: process = subprocess.Popen(cfx, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) response = process.communicate() except subprocess.CalledProcessError, err: info_write(info_targetpath, "error", str(err), hashtag) log.critical("[xpi:%s] Failed to build xpi: %s. Command(%s)" % (hashtag, str(err), cfx)) shutil.rmtree(temp_dir) raise
def copy(request, revision_id): """ Copy package - create a duplicate of the Package, set user as author """ source = get_object_with_related_or_404(PackageRevision, pk=revision_id) log.debug("[copy: %s] Copying started from (%s)" % (revision_id, source)) # save package try: package = source.package.copy(request.user) except IntegrityError, err: log.critical(("[copy: %s] Package copy failed") % revision_id) return HttpResponseForbidden("You already have a %s with that name" % escape(source.package.get_type_name()))
def copy(request, revision_id): """ Copy package - create a duplicate of the Package, set user as author """ source = get_object_with_related_or_404(PackageRevision, pk=revision_id) log.debug('[copy: %s] Copying started from (%s)' % (revision_id, source)) # save package try: package = source.package.copy(request.user) except IntegrityError, err: log.critical(("[copy: %s] Package copy failed") % revision_id) return HttpResponseForbidden('You already have a %s with that name' % escape(source.package.get_type_name()))
def build(sdk_dir, package_dir, filename, hashtag): """Build xpi from source in sdk_dir.""" t1 = time.time() # create XPI os.chdir(package_dir) # @TODO xulrunner should be a config variable cfx = [settings.PYTHON_EXEC, '%s/bin/cfx' % sdk_dir, '--binary=/usr/bin/xulrunner', '--keydir=%s/%s' % (sdk_dir, settings.KEYDIR), 'xpi'] env = dict(PATH='%s/bin:%s' % (sdk_dir, os.environ['PATH']), VIRTUAL_ENV=sdk_dir, CUDDLEFISH_ROOT=sdk_dir, PYTHONPATH=os.path.join(sdk_dir, 'python-lib')) try: process = subprocess.Popen(cfx, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) response = process.communicate() except subprocess.CalledProcessError: log.critical("Failed to build xpi: %s. Command(%s)" % ( subprocess.CalledProcessError, cfx)) return HttpResponseServerError if response[1]: log.critical("Failed to build xpi.\nError: %s" % response[1]) return HttpResponseForbidden(response[1]) # move the XPI created to the XPI_TARGETDIR xpi_targetfilename = "%s.xpi" % hashtag xpi_targetpath = os.path.join(settings.XPI_TARGETDIR, xpi_targetfilename) xpi_path = os.path.join(package_dir, "%s.xpi" % filename) log.debug(response) log.debug("%s, %s" % (xpi_path, xpi_targetpath)) shutil.copy(xpi_path, xpi_targetpath) shutil.rmtree(sdk_dir) ret = [xpi_targetfilename] ret.extend(response) t2 = time.time() log.info('[xpi:%s] Created xpi: %s (time: %0.3fms)' % (hashtag, xpi_targetpath, ((t2 - t1) * 1000))) return ret
def update_global_totals(job, date, **kw): log.info('Updating global statistics totals (%s) for (%s)' % (job, date)) jobs = _get_daily_jobs(date) jobs.update(_get_metrics_jobs(date)) num = jobs[job]() q = """REPLACE INTO global_stats (`name`, `count`, `date`) VALUES (%s, %s, %s)""" p = [job, num or 0, date] try: cursor = connection.cursor() cursor.execute(q, p) except Exception, e: log.critical('Failed to update global stats: (%s): %s' % (p, e))
def build(sdk_dir, package_dir, filename, hashtag, tstart=None): """Build xpi from SDK with prepared packages in sdk_dir. :params: * sdk_dir (String) SDK directory * package_dir (string) dir of the Add-on package * filename (string) XPI will be build with this name * hashtag (string) XPI will be copied to a file which name is creted using the unique hashtag * t1 (integer) time.time() of the process started :returns: (list) ``cfx xpi`` response where ``[0]`` is ``stdout`` and ``[1]`` ``stderr`` """ t1 = time.time() # create XPI os.chdir(package_dir) # @TODO xulrunner should be a config variable cfx = [settings.PYTHON_EXEC, '%s/bin/cfx' % sdk_dir, '--binary=/usr/bin/xulrunner', '--keydir=%s/%s' % (sdk_dir, settings.KEYDIR), 'xpi', '--strip-xpi'] log.debug(cfx) info_targetfilename = "%s.json" % hashtag info_targetpath = os.path.join(settings.XPI_TARGETDIR, info_targetfilename) env = dict(PATH='%s/bin:%s' % (sdk_dir, os.environ['PATH']), VIRTUAL_ENV=sdk_dir, CUDDLEFISH_ROOT=sdk_dir, PYTHONPATH=os.path.join(sdk_dir, 'python-lib')) try: process = subprocess.Popen(cfx, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) response = process.communicate() except subprocess.CalledProcessError, err: info_write(info_targetpath, 'error', str(err), hashtag) log.critical("[xpi:%s] Failed to build xpi: %s. Command(%s)" % ( hashtag, str(err), cfx)) shutil.rmtree(sdk_dir) raise
def update_webtrend(url, date, **kw): resp = requests.get(url, auth=(settings.WEBTRENDS_USERNAME, settings.WEBTRENDS_PASSWORD)) if resp.status_code != 200: log.critical('Failed to fetch webtrends stats. url: %s status_code: %s' % (url, resp.status_code)) return else: data = resp.json key, val = data['data'][0]['measures'].items()[0] p = ['webtrends_' + key, val, date] try: cursor = connection.cursor() cursor.execute('REPLACE INTO global_stats (name, count, date) ' 'values (%s, %s, %s)', p) transaction.commit_unless_managed() except Exception, e: log.critical('Failed to update global stats: (%s): %s' % (p, e))
def update_global_totals(job, date, **kw): log.info("[%s] Updating global statistics totals (%s) for (%s)" % (update_global_totals.rate_limit, job, date)) jobs = _get_daily_jobs(date) jobs.update(_get_metrics_jobs(date)) num = jobs[job]() q = """REPLACE INTO global_stats(`name`, `count`, `date`) VALUES (%s, %s, %s)""" p = [job, num or 0, date] try: cursor = connection.cursor() cursor.execute(q, p) transaction.commit_unless_managed() except Exception, e: log.critical("Failed to update global stats: (%s): %s" % (p, e))
def all_zip(request, pk): """Zip all and return a file.""" if not pk: log.critical("[zip] No package_id provided") return package = Package.objects.get(pk=pk) zips = [] # Zip all revisions of the package for revision in package.revisions.all(): zips.append(revision.zip_source(hashtag=revision.get_cache_hashtag())) # Zip all zipped revisions into one file zip_targetname = "package-%d.zip" % package.pk zip_targetpath = os.path.join(settings.XPI_TARGETDIR, zip_targetname) with closing(ZipFile(zip_targetpath, 'w', ZIP_DEFLATED)) as z: for fn in zips: z.write(fn, os.path.basename(fn)) log.info('[zipall:%s] Downloading All zipped' % pk) response = serve(request, zip_targetpath, '/', show_indexes=False) response['Content-Disposition'] = ('attachment; filename="%s"' % zip_targetname) return response
def update_google_analytics(date, **kw): creds_data = getattr(settings, 'GOOGLE_ANALYTICS_CREDENTIALS', None) if not creds_data: log.critical('Failed to update global stats: ' 'GOOGLE_ANALYTICS_CREDENTIALS not set') return creds = OAuth2Credentials( *[creds_data[k] for k in ('access_token', 'client_id', 'client_secret', 'refresh_token', 'token_expiry', 'token_uri', 'user_agent')]) h = httplib2.Http() creds.authorize(h) service = build('analytics', 'v3', http=h) domain = getattr(settings, 'GOOGLE_ANALYTICS_DOMAIN', None) or settings.DOMAIN profile_id = get_profile_id(service, domain) if profile_id is None: log.critical('Failed to update global stats: could not access a Google' ' Analytics profile for ' + domain) return datestr = date.strftime('%Y-%m-%d') try: data = service.data().ga().get(ids='ga:' + profile_id, start_date=datestr, end_date=datestr, metrics='ga:visits').execute() # Storing this under the webtrends stat name so it goes on the # same graph as the old webtrends data. p = ['webtrends_DailyVisitors', data['rows'][0][0], date] except Exception, e: log.critical( 'Fetching stats data for %s from Google Analytics failed: %s' % e) return
def xpi_build_from_model(rev_pk, mod_codes={}, att_codes={}, hashtag=None): """ Get object and build xpi """ if not hashtag: log.critical("No hashtag provided") return revision = PackageRevision.objects.get(pk=rev_pk) # prepare changed modules and attachments modules = [] attachments = [] for mod in revision.modules.all(): if str(mod.pk) in mod_codes: mod.code = mod_codes[str(mod.pk)] modules.append(mod) for att in revision.attachments.all(): if str(att.pk) in att_codes: att.code = att_codes[str(att.pk)] attachments.append(att) revision.build_xpi( modules=modules, attachments=attachments, hashtag=hashtag)
for name in ['doc', 'lib', 'data', 'tests']: _extract(f, name, resource_dir_prefix) # Add all dependencies to the manifest # This is a flat list - hierarchy might be different from original self.manifest['dependencies'].extend(dependencies) # Add icon files to manifest for f in main_dir_files: if 'icon' in f: self.manifest[f.split('.')[0]] = f # create add-on's package.json file log.debug('Writing manifest %s, %s' % (os.path.join( sdk_dir, 'packages', package_name, 'package.json'), self.manifest)) package_dir = os.path.join( sdk_dir, 'packages', package_name) try: with open(os.path.join(package_dir, 'package.json'), 'w') as manifest: manifest.write(simplejson.dumps(self.manifest)) except: log.critical("Manifest couldn't be exported to %s" % package_path) raise return sdk_dir def cleanup(self): """closes all files opened during the repackaging """ self.xpi_zip.close() self.xpi_temp.close()
end_date=datestr, metrics='ga:visits').execute() # Storing this under the webtrends stat name so it goes on the # same graph as the old webtrends data. p = ['webtrends_DailyVisitors', data['rows'][0][0], date] except Exception, e: log.critical( 'Fetching stats data for %s from Google Analytics failed: %s' % e) return try: cursor = connection.cursor() cursor.execute('REPLACE INTO global_stats (name, count, date) ' 'values (%s, %s, %s)', p) transaction.commit_unless_managed() except Exception, e: log.critical('Failed to update global stats: (%s): %s' % (p, e)) return log.debug('Committed global stats details: (%s) has (%s) for (%s)' % tuple(p)) @task def update_global_totals(job, date, **kw): log.info("Updating global statistics totals (%s) for (%s)" % (job, date)) jobs = _get_daily_jobs(date) jobs.update(_get_metrics_jobs(date)) num = jobs[job]()
# Check memcached memcache = getattr(settings, 'CACHES', {}).get('default') memcache_results = [] if memcache and 'memcached' in memcache['BACKEND']: hosts = memcache['LOCATION'] if not isinstance(hosts, (tuple, list)): hosts = [hosts] for host in hosts: ip, port = host.split(':') try: s = socket.socket() s.connect((ip, int(port))) except Exception, e: status = False result = False log.critical('Failed to connect to memcached (%s): %s' % (host, e)) else: result = True finally: s.close() memcache_results.append((ip, port, result)) if len(memcache_results) < 2: status = False log.warning('You should have 2+ memcache servers. ' 'You have %d.' % len(memcache_results)) if not memcache_results: status = False log.info('Memcached is not configured.') data['memcached'] = memcache_results
def build(sdk_dir, package_dir, filename, hashtag, tstart=None, options=None, temp_dir=None): """Build xpi from SDK with prepared packages in sdk_dir. :params: * sdk_dir (String) SDK directory * package_dir (string) dir of the Add-on package * filename (string) XPI will be build with this name * hashtag (string) XPI will be copied to a file which name is creted using the unique hashtag * t1 (integer) time.time() of the process started :returns: (list) ``cfx xpi`` response where ``[0]`` is ``stdout`` and ``[1]`` ``stderr`` """ t1 = time.time() if not temp_dir: temp_dir = sdk_dir # create XPI os.chdir(package_dir) cfx = [ settings.PYTHON_EXEC, '%s/bin/cfx' % sdk_dir, '--binary=%s' % settings.XULRUNNER_BINARY, '--keydir=%s/%s' % (sdk_dir, settings.KEYDIR), 'xpi' ] if options: cfx.append(options) log.debug(cfx) info_targetfilename = "%s.json" % hashtag info_targetpath = os.path.join(settings.XPI_TARGETDIR, info_targetfilename) env = dict(PATH='%s/bin:%s' % (sdk_dir, os.environ['PATH']), DISPLAY=os.environ.get('DISPLAY', ':0'), VIRTUAL_ENV=sdk_dir, CUDDLEFISH_ROOT=sdk_dir, PYTHONPATH=os.path.join(sdk_dir, 'python-lib')) try: process = subprocess.Popen(cfx, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) response = process.communicate() except subprocess.CalledProcessError, err: info_write(info_targetpath, 'error', str(err), hashtag) log.critical("[xpi:%s] Failed to build xpi: %s. Command(%s)" % (hashtag, str(err), cfx)) shutil.rmtree(temp_dir) raise
def extract_po(project, locale, path, entities=False): """Extract .po (gettext) file with path and save or update in DB.""" try: po = polib.pofile(path) relative_path = get_relative_path(path, locale) if relative_path[-1] == 't': relative_path = relative_path[:-1] resource, created = Resource.objects.get_or_create(project=project, path=relative_path, format='po') if entities: for order, entry in enumerate(po): if not entry.obsolete: save_entity(resource=resource, string=entry.msgid, string_plural=entry.msgid_plural, comment=entry.comment, order=order, source=entry.occurrences) update_entity_count(resource) else: for entry in (po.translated_entries() + po.fuzzy_entries()): if not entry.obsolete: # Entities without plurals if len(entry.msgstr) > 0: try: e = Entity.objects.get(resource=resource, string=entry.msgid) save_translation(entity=e, locale=locale, string=entry.msgstr, fuzzy='fuzzy' in entry.flags) except Entity.DoesNotExist: continue # Pluralized entities elif len(entry.msgstr_plural) > 0: try: e = Entity.objects.get(resource=resource, string=entry.msgid) for k in entry.msgstr_plural: save_translation(entity=e, locale=locale, string=entry.msgstr_plural[k], plural_form=k, fuzzy='fuzzy' in entry.flags) except Entity.DoesNotExist: continue update_stats(resource, locale) log.debug("[" + locale.code + "]: " + path + " saved to DB.") except Exception as e: log.critical('PoExtractError for %s: %s' % (path, e))
:attr: amo_id (int) id of the add-on in AMO :attr: amo_file_id (int) id of the file uploaded to AMO :returns: dict """ url = get_addon_amo_api_url(amo_id, amo_file_id) log.debug("AMOAPI: receiving add-on info from \"%s\"" % url) req = urllib2.Request(url) try: page = urllib2.urlopen(req, timeout=settings.URLOPEN_TIMEOUT) except urllib2.HTTPError, error: if '404' in str(error): return {'deleted': True} except Exception, error: msg = "AMOAPI: ERROR receiving add-on info from \"%s\"%s%s" log.critical(msg % (url, '\n', str(error))) return {'error': msg % (url, ' : ', str(error))} amo_xml = etree.fromstring(page.read()) amo_data = {} for element in amo_xml.iter(): if element.tag in ('status', 'rating', 'version', 'slug'): amo_data[element.tag] = element.text if element.tag == 'status': amo_data['status_code'] = int(element.get('id')) # return dict return amo_data def fetch_amo_user(email): log.debug('#'*80) amo = AMOOAuth(domain=settings.AMOOAUTH_DOMAIN,
except subprocess.CalledProcessError, err: info_write(info_targetpath, 'error', str(err), hashtag) log.critical("[xpi:%s] Failed to build xpi: %s. Command(%s)" % (hashtag, str(err), cfx)) shutil.rmtree(temp_dir) raise if (waffle.switch_is_active('SDKErrorInStdOutWorkaround') and not os.path.exists( os.path.join(package_dir, '%s.xpi' % filename))): badresponse = response[0] response = ['', ''] response[1] = badresponse xpi_path = os.path.join(package_dir, "%s.xpi" % filename) if process.returncode != 0: info_write(info_targetpath, 'error', response[1], hashtag) log.critical("[xpi:%s] Failed to build xpi., stderr: %s" % (hashtag, response[1])) shutil.rmtree(temp_dir) return response t2 = time.time() # XPI: move the XPI created to the XPI_TARGETDIR (local to NFS) xpi_targetfilename = "%s.xpi" % hashtag xpi_targetpath = os.path.join(settings.XPI_TARGETDIR, xpi_targetfilename) try: shutil.copy(xpi_path, xpi_targetpath) except IOError, err: info_write(info_targetpath, 'error', 'XPI file can not be copied.', hashtag) log.critical("[xpi:%s] Failed to copy xpi.\n%s" % (hashtag, str(err))) shutil.rmtree(temp_dir)
VIRTUAL_ENV=sdk_dir, CUDDLEFISH_ROOT=sdk_dir, PYTHONPATH=os.path.join(sdk_dir, 'python-lib')) try: process = subprocess.Popen(cfx, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) response = process.communicate() except subprocess.CalledProcessError, err: info_write(info_targetpath, 'error', str(err), hashtag) log.critical("[xpi:%s] Failed to build xpi: %s. Command(%s)" % ( hashtag, str(err), cfx)) shutil.rmtree(sdk_dir) raise if response[1]: info_write(info_targetpath, 'error', response[1], hashtag) log.critical("[xpi:%s] Failed to build xpi." % hashtag) shutil.rmtree(sdk_dir) return response t2 = time.time() # XPI: move the XPI created to the XPI_TARGETDIR (local to NFS) xpi_path = os.path.join(package_dir, "%s.xpi" % filename) xpi_targetfilename = "%s.xpi" % hashtag xpi_targetpath = os.path.join(settings.XPI_TARGETDIR, xpi_targetfilename) shutil.copy(xpi_path, xpi_targetpath) shutil.rmtree(sdk_dir) ret = [xpi_targetfilename] ret.extend(response)
:attr: amo_id (int) id of the add-on in AMO :attr: amo_file_id (int) id of the file uploaded to AMO :returns: dict """ url = get_addon_amo_api_url(amo_id, amo_file_id) log.debug("AMOAPI: receiving add-on info from \"%s\"" % url) req = urllib2.Request(url) try: page = urllib2.urlopen(req, timeout=settings.URLOPEN_TIMEOUT) except urllib2.HTTPError, error: if '404' in str(error): return {'deleted': True} except Exception, error: msg = "AMOAPI: ERROR receiving add-on info from \"%s\"%s%s" log.critical(msg % (url, '\n', str(error))) return {'error': msg % (url, ' : ', str(error))} amo_xml = etree.fromstring(page.read()) amo_data = {} for element in amo_xml.iter(): if element.tag in ('status', 'rating', 'version', 'slug'): amo_data[element.tag] = element.text if element.tag == 'status': amo_data['status_code'] = int(element.get('id')) # return dict return amo_data def fetch_amo_user(email): log.debug('#' * 80) amo = AMOOAuth(domain=settings.AMOOAUTH_DOMAIN,
resource_dir_prefix = uri_prefix_1 for name in ['docs', 'lib', 'data', 'tests']: _extract(f, name, resource_dir_prefix) # Add all dependencies to the manifest # This is a flat list - hierarchy might be different from original self.manifest['dependencies'].extend(dependencies) # Add icon files to manifest for f in main_dir_files: if 'icon' in f: self.manifest[f.split('.')[0]] = f # create add-on's package.json file log.debug('Writing manifest %s, %s' % (os.path.join( sdk_dir, 'packages', package_name, 'package.json'), self.manifest)) package_dir = os.path.join(sdk_dir, 'packages', package_name) try: with open(os.path.join(package_dir, 'package.json'), 'w') as manifest: manifest.write(simplejson.dumps(self.manifest)) except: log.critical("Manifest couldn't be exported to %s" % package_path) raise return sdk_dir def cleanup(self): """closes all files opened during the repackaging """ self.xpi_zip.close() self.xpi_temp.close()
metrics='ga:visits').execute() # Storing this under the webtrends stat name so it goes on the # same graph as the old webtrends data. p = ['webtrends_DailyVisitors', data['rows'][0][0], date] except Exception, e: log.critical( 'Fetching stats data for %s from Google Analytics failed: %s' % e) return try: cursor = connection.cursor() cursor.execute( 'REPLACE INTO global_stats (name, count, date) ' 'values (%s, %s, %s)', p) transaction.commit_unless_managed() except Exception, e: log.critical('Failed to update global stats: (%s): %s' % (p, e)) return log.debug('Committed global stats details: (%s) has (%s) for (%s)' % tuple(p)) @task def update_global_totals(job, date, **kw): log.info("Updating global statistics totals (%s) for (%s)" % (job, date)) jobs = _get_daily_jobs(date) jobs.update(_get_metrics_jobs(date)) num = jobs[job]()
metrics='ga:visits').execute() # Storing this under the webtrends stat name so it goes on the # same graph as the old webtrends data. p = ['webtrends_DailyVisitors', data['rows'][0][0], date] except Exception, e: log.critical( 'Fetching stats data for %s from Google Analytics failed: %s' % e) return try: cursor = connection.cursor() cursor.execute( 'REPLACE INTO global_stats (name, count, date) ' 'values (%s, %s, %s)', p) transaction.commit_unless_managed() except Exception, e: log.critical('Failed to update global stats: (%s): %s' % (p, e)) return log.debug('Committed global stats details: (%s) has (%s) for (%s)' % tuple(p)) @task def update_global_totals(job, date, **kw): log.info('Updating global statistics totals (%s) for (%s)' % (job, date)) jobs = _get_daily_jobs(date) jobs.update(_get_metrics_jobs(date)) num = jobs[job]()
end_date=datestr, metrics='ga:visits').execute() # Storing this under the webtrends stat name so it goes on the # same graph as the old webtrends data. p = ['webtrends_DailyVisitors', data['rows'][0][0], date] except Exception, e: log.critical( 'Fetching stats data for %s from Google Analytics failed: %s' % e) return try: cursor = connection.cursor() cursor.execute('REPLACE INTO global_stats (name, count, date) ' 'values (%s, %s, %s)', p) transaction.commit_unless_managed() except Exception, e: log.critical('Failed to update global stats: (%s): %s' % (p, e)) return log.debug('Committed global stats details: (%s) has (%s) for (%s)' % tuple(p)) @task def update_global_totals(job, date, **kw): log.info('Updating global statistics totals (%s) for (%s)' % (job, date)) jobs = _get_daily_jobs(date) jobs.update(_get_metrics_jobs(date)) num = jobs[job]()
response = process.communicate() except subprocess.CalledProcessError, err: info_write(info_targetpath, 'error', str(err), hashtag) log.critical("[xpi:%s] Failed to build xpi: %s. Command(%s)" % ( hashtag, str(err), cfx)) shutil.rmtree(temp_dir) raise if (waffle.switch_is_active('SDKErrorInStdOutWorkaround') and not os.path.exists(os.path.join(package_dir, '%s.xpi' % filename))): badresponse = response[0] response = ['', ''] response[1] = badresponse xpi_path = os.path.join(package_dir, "%s.xpi" % filename) if process.returncode != 0: info_write(info_targetpath, 'error', response[1], hashtag) log.critical("[xpi:%s] Failed to build xpi., stderr: %s" % ( hashtag, response[1])) shutil.rmtree(temp_dir) return response t2 = time.time() # XPI: move the XPI created to the XPI_TARGETDIR (local to NFS) xpi_targetfilename = "%s.xpi" % hashtag xpi_targetpath = os.path.join(settings.XPI_TARGETDIR, xpi_targetfilename) try: shutil.copy(xpi_path, xpi_targetpath) except IOError, err: info_write(info_targetpath, 'error', 'XPI file can not be copied.', hashtag) log.critical("[xpi:%s] Failed to copy xpi.\n%s" % (hashtag, str(err)))
stderr=subprocess.PIPE, env=env) response = process.communicate() except subprocess.CalledProcessError, err: info_write(info_targetpath, 'error', str(err), hashtag) log.critical("[xpi:%s] Failed to build xpi: %s. Command(%s)" % ( hashtag, str(err), cfx)) shutil.rmtree(sdk_dir) raise if (waffle.switch_is_active('SDKErrorInStdOutWorkaround') and not os.path.exists(os.path.join(package_dir, '%s.xpi' % filename))): badresponse = response[0] response = ['', ''] response[1] = badresponse if response[1]: info_write(info_targetpath, 'error', response[1], hashtag) log.critical("[xpi:%s] Failed to build xpi." % hashtag) shutil.rmtree(sdk_dir) return response t2 = time.time() # XPI: move the XPI created to the XPI_TARGETDIR (local to NFS) xpi_path = os.path.join(package_dir, "%s.xpi" % filename) xpi_targetfilename = "%s.xpi" % hashtag xpi_targetpath = os.path.join(settings.XPI_TARGETDIR, xpi_targetfilename) try: shutil.copy(xpi_path, xpi_targetpath) except IOError, err: info_write(info_targetpath, 'error', 'XPI file can not be copied.', hashtag)
start_date=datestr, end_date=datestr, metrics='ga:visits').execute() # Storing this under the webtrends stat name so it goes on the # same graph as the old webtrends data. p = ['webtrends_DailyVisitors', data['rows'][0][0], date] except Exception, e: log.critical( 'Fetching stats data for %s from Google Analytics failed: %s' % e) return try: cursor = connection.cursor() cursor.execute('REPLACE INTO global_stats (name, count, date) ' 'values (%s, %s, %s)', p) except Exception, e: log.critical('Failed to update global stats: (%s): %s' % (p, e)) return log.debug('Committed global stats details: (%s) has (%s) for (%s)' % tuple(p)) @task def update_global_totals(job, date, **kw): log.info('Updating global statistics totals (%s) for (%s)' % (job, date)) jobs = _get_daily_jobs(date) jobs.update(_get_metrics_jobs(date)) num = jobs[job]()
log.debug(cfx) env = dict(PATH='%s/bin:%s' % (sdk_dir, os.environ['PATH']), VIRTUAL_ENV=sdk_dir, CUDDLEFISH_ROOT=sdk_dir, PYTHONPATH=os.path.join(sdk_dir, 'python-lib')) try: process = subprocess.Popen(cfx, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) response = process.communicate() except subprocess.CalledProcessError, err: log.critical("Failed to build xpi: %s. Command(%s)" % ( str(err), cfx)) raise if response[1] and not force_guid: log.critical("Failed to build xpi.\nError: %s" % response[1]) return response xpi_path = os.path.join(package_dir, "%s.xpi" % filename) # move the XPI created to the XPI_TARGETDIR xpi_targetfilename = "%s.xpi" % hashtag xpi_targetpath = os.path.join(settings.XPI_TARGETDIR, xpi_targetfilename) shutil.copy(xpi_path, xpi_targetpath) shutil.rmtree(sdk_dir) ret = [xpi_targetfilename] ret.extend(response) t2 = time.time()
def extract_po(project, locale, paths, entities=False): """Extract .po (gettext) files from paths and save or update in DB.""" for path in paths: try: po = polib.pofile(path) escape = polib.escape relative_path = get_relative_path(path, locale) if relative_path[-1] == 't': relative_path = relative_path[:-1] resource, created = Resource.objects.get_or_create( project=project, path=relative_path) if entities: for entry in po: if not entry.obsolete: save_entity(resource=resource, string=escape(entry.msgid), string_plural=escape(entry.msgid_plural), comment=entry.comment, source=entry.occurrences) update_entity_count(resource, project) else: for entry in (po.translated_entries() + po.fuzzy_entries()): if not entry.obsolete: # Entities without plurals if len(escape(entry.msgstr)) > 0: try: e = Entity.objects.get( resource=resource, string=escape(entry.msgid)) save_translation( entity=e, locale=locale, string=escape(entry.msgstr), fuzzy='fuzzy' in entry.flags) except Entity.DoesNotExist: continue # Pluralized entities elif len(entry.msgstr_plural) > 0: try: e = Entity.objects.get( resource=resource, string=escape(entry.msgid)) for k in entry.msgstr_plural: save_translation( entity=e, locale=locale, string=escape(entry.msgstr_plural[k]), plural_form=k, fuzzy='fuzzy' in entry.flags) except Entity.DoesNotExist: continue update_stats(resource, locale) log.debug("[" + locale.code + "]: " + path + " saved to DB.") except Exception as e: log.critical('PoExtractError for %s: %s' % (path, e))