Пример #1
0
def list_job(request):
    cclient = chronos.connect('{}:{}'.format(settings.CHRONOS['host'],
                                             settings.CHRONOS['port']))
    jobs = cclient.list()
    for job in jobs:
        last_success = job['lastSuccess']
        last_error = job['lastError']

        if last_success == "" and last_error != "":
            job['last'] = "error"
        elif last_success != "" and last_error == "":
            job['last'] = "success"
        elif last_success != "" and last_error != "":
            last_success = dateutil.parser.parse(job['lastSuccess'])
            last_error = dateutil.parser.parse(job['lastError'])
            if last_error > last_success:
                job['last'] = "error"
            else:
                job['last'] = "success"
        elif last_success == "" and last_error == "":
            job['last'] = "-"

        job['stat'] = cclient.job_stat(job['name'])['histogram']
    data = {'jobs': jobs}
    data['refresh'] = 3000
    return render(request, 'chronos_mgmt/list_job.html', data)
Пример #2
0
def get_stars_client():
    # store client in static function var, so there's only one instance
    if not hasattr(get_stars_client, "stars_client"):
        services_endpoint = os.getenv("DATACOMMONS_SERVICES_ENDPOINT")
        c = os.getenv("CHRONOS_URL")
        if not c:
            raise RuntimeError("The datacommons module requires CHRONOS_URL to be set")
        urlp = six.moves.urllib.parse.urlparse(c)
        if urlp.scheme:
            chronos_proto = urlp.scheme
            chronos_endpoint = c[len(urlp.scheme) + 3:]
        else:
            chronos_proto = "https"
            chronos_endpoint = c

        get_stars_client.stars_client = Stars(
            services_endpoints  = services_endpoint.split(",") if services_endpoint else None,
            scheduler_endpoints = chronos_endpoint.split(",") if chronos_endpoint else None)

        # override chronos client to use proto set in DATACOMMONS_CHRONOS_PROTO
        # TODO update stars to allow proto specification
        # https://github.com/stevencox/cwltool/issues/14
        get_stars_client.stars_client.scheduler.client = chronos.connect(chronos_endpoint, proto=chronos_proto)

    return get_stars_client.stars_client
Пример #3
0
def list_job(request):
    cclient = chronos.connect("{}:{}".format(settings.CHRONOS["host"], settings.CHRONOS["port"]))
    jobs = cclient.list()
    for job in jobs:
        last_success = job["lastSuccess"]
        last_error = job["lastError"]

        if last_success == "" and last_error != "":
            job["last"] = "error"
        elif last_success != "" and last_error == "":
            job["last"] = "success"
        elif last_success != "" and last_error != "":
            last_success = dateutil.parser.parse(job["lastSuccess"])
            last_error = dateutil.parser.parse(job["lastError"])
            if last_error > last_success:
                job["last"] = "error"
            else:
                job["last"] = "success"
        elif last_success == "" and last_error == "":
            job["last"] = "-"

        job["stat"] = cclient.job_stat(job["name"])["histogram"]
    data = {"jobs": jobs}
    data["refresh"] = 3000
    return render(request, "chronos_mgmt/list_job.html", data)
Пример #4
0
def list_job(request):
    cclient = chronos.connect('{}:{}'.format(settings.CHRONOS['host'], settings.CHRONOS['port']))
    jobs = cclient.list()
    for job in jobs:
        last_success = job['lastSuccess']
        last_error = job['lastError']

        if last_success == "" and last_error != "":
            job['last'] = "error"
        elif last_success != "" and last_error == "":
            job['last'] = "success"
        elif last_success != "" and last_error != "":
            last_success = dateutil.parser.parse(job['lastSuccess'])
            last_error = dateutil.parser.parse(job['lastError'])
            if last_error > last_success:
                job['last'] = "error"
            else:
                job['last'] = "success"
        elif last_success == "" and last_error == "":
            job['last'] = "-"

        job['stat'] = cclient.job_stat(job['name'])['histogram']
    data = {'jobs': jobs}
    data['refresh'] = 3000
    return render(request, 'chronos_mgmt/list_job.html', data)
Пример #5
0
def get_chronos_client(config):
    """Returns a chronos client object for interacting with the API"""
    chronos_hosts = config.get_url()
    chronos_hostnames = [urlparse.urlsplit(hostname).netloc for hostname in chronos_hosts]
    log.info("Attempting to connect to Chronos servers: %s" % chronos_hosts)
    return chronos.connect(servers=chronos_hostnames,
                           username=config.get_username(),
                           password=config.get_password())
Пример #6
0
def get_chronos_client(config):
    """Returns a chronos client object for interacting with the API"""
    chronos_hosts = config.get_url()
    chronos_hostnames = [urlparse.urlsplit(hostname).netloc for hostname in chronos_hosts]
    log.info("Attempting to connect to Chronos servers: %s" % chronos_hosts)
    return chronos.connect(servers=chronos_hostnames,
                           username=config.get_username(),
                           password=config.get_password())
Пример #7
0
def main():
    parser = argparse.ArgumentParser(description="Monitor the status of Chronos Jobs")
    parser.add_argument("--hostname", metavar="<host:port>", required=True,
                        help="hostname and port of the Chronos instance")
    parser.add_argument("--prefix", metavar="job-prefix", required=False, action="append",
                        help="if set, only check jobs matching this prefix")
    parser.add_argument("--exclude", metavar="job-prefix", required=False, action="append",
                        help="if set, exclude jobs matching this prefix")
    parser.add_argument("--warn", metavar="#", default=1,
                        help="warn if at least this number of jobs are currently failed")
    parser.add_argument("--crit", metavar="#", default=1,
                        help="critical if at least this number of jobs are currently failed")
    args = parser.parse_args()

    fails = []
    ok = []
    unknown = []

    c = chronos.connect(args.hostname)
    cjobs = c.list()

    if not isinstance(cjobs, list):
        print "UNKNOWN: error querying chronos"
        sys.exit(3)

    for job in cjobs:
        if job['disabled']:
            continue

        if isinstance(args.prefix, list):
            if not match_prefix(args.prefix, job['name']):
                continue

        if isinstance(args.exclude, list):
            if match_prefix(args.exclude, job['name']):
                continue

        if job['lastError'] > job['lastSuccess']:
            fails.append(job['name'].encode('ascii'))
        elif job['lastSuccess']:
            ok.append(job['name'].encode('ascii'))
        else:
            unknown.append(job['name'].encode('ascii'))

    if len(unknown) > 0:
        umsg = "(%d waiting for execution or with no data)" % len(unknown)
    else:
        umsg = ''

    if len(fails) == 0:
        print "OK: %d jobs succeeded on last run %s" % (len(ok), umsg)
        sys.exit(0)
    elif len(fails) >= int(args.crit):
        print "CRITICAL: %d failed jobs: %s %s" % (len(fails), str(fails).strip('[]'), umsg)
        sys.exit(2)
    elif len(fails) >= int(args.warn):
        print "WARNING: %d failed jobs: %s %s" % (len(fails), str(fails).strip('[]'), umsg)
        sys.exit(1)
Пример #8
0
def dashboard(request):
    data = {}
    data['total_template'] = Template.objects.count()
    mc = MarathonClient('http://{}:{}'.format(settings.MARATHON['host'], settings.MARATHON['port']))
    data['total_app'] = len(mc.list_apps())
    cclient = chronos.connect('{}:{}'.format(settings.CHRONOS['host'], settings.CHRONOS['port']))
    jobs = cclient.list()
    data['total_job'] = len(cclient.list())
    data['total_watcher'] = len(settings.WATCHER_THREADS)
    return render(request, 'dashboard/dashboard.html',data)
def working_chronos(context):
    """Adds a working chronos client as context.client for the purposes of
    interacting with it in the test."""
    if not hasattr(context, 'client'):
        chronos_servers = ['127.0.0.1:4400']
        chronos_version = context.config.userdata.get('chronos_version', DEFAULT_CHRONOS_VERSION)
        if chronos_version in LEGACY_VERSIONS:
            scheduler_api_version = None
        else:
            scheduler_api_version = 'v1'
        context.client = chronos.connect(chronos_servers, scheduler_api_version=scheduler_api_version)
Пример #10
0
 def __init__(self, app, nworkers, **kwargs):
     """Initialize this job runner and start the monitor thread"""
     assert chronos, CHRONOS_IMPORT_MSG
     if self.RUNNER_PARAM_SPEC_KEY not in kwargs:
         kwargs[self.RUNNER_PARAM_SPEC_KEY] = {}
     kwargs[self.RUNNER_PARAM_SPEC_KEY].update(self.RUNNER_PARAM_SPEC)
     super().__init__(app, nworkers, **kwargs)
     protocol = 'http' if self.runner_params.get('insecure',
                                                 True) else 'https'
     self._chronos_client = chronos.connect(
         self.runner_params['chronos'],
         username=self.runner_params.get('username'),
         password=self.runner_params.get('password'),
         proto=protocol)
Пример #11
0
 def __init__(self, app, nworkers, **kwargs):
     """Initialize this job runner and start the monitor thread"""
     assert chronos, CHRONOS_IMPORT_MSG
     if self.RUNNER_PARAM_SPEC_KEY not in kwargs:
         kwargs[self.RUNNER_PARAM_SPEC_KEY] = {}
     kwargs[self.RUNNER_PARAM_SPEC_KEY].update(self.RUNNER_PARAM_SPEC)
     super(ChronosJobRunner, self).__init__(app, nworkers, **kwargs)
     protocol = 'http' if self.runner_params.get('insecure', True) else 'https'
     self._chronos_client = chronos.connect(
         self.runner_params['chronos'],
         username=self.runner_params.get('username'),
         password=self.runner_params.get('password'),
         proto=protocol)
     self._init_monitor_thread()
     self._init_worker_threads()
Пример #12
0
def dashboard(request):
    data = {}
    data["total_template"] = Template.objects.count()
    try:
        mc = MarathonClient("http://{}:{}".format(settings.MARATHON["host"], settings.MARATHON["port"]))
        data["total_app"] = len(mc.list_apps())
    except Exception as e:
        data["total_app"] = []
    try:
        cclient = chronos.connect("{}:{}".format(settings.CHRONOS["host"], settings.CHRONOS["port"]))
        jobs = cclient.list()
        data["total_job"] = len(cclient.list())
    except Exception as e:
        data["total_job"] = []

    data["total_watcher"] = len(settings.WATCHER_THREADS)
    return render(request, "dashboard/dashboard.html", data)
Пример #13
0
def send_to_chronos(request):
    try:
        if request.method == "POST":
            action = request.POST.get("action", None)
            job_name = request.POST.get("name", None)
            cclient = chronos.connect("{}:{}".format(settings.CHRONOS["host"], settings.CHRONOS["port"]))
            if action == "destroy":
                cclient.delete(job_name)
            elif action == "run":
                cclient.run(job_name)

            result = '{"status":"success", "msg": "%(action)s success"}' % {"action": action}
    except Exception as e:
        result = '{"status":"error", "msg": "%(action)s fail: %(error)s" }' % {
            "action": action,
            "error": html.escape(str(e)),
        }
    return HttpResponse(result)
Пример #14
0
def send_to_chronos(request):
    try:
        if request.method == 'POST':
            action = request.POST.get('action', None)
            job_name = request.POST.get('name', None)
            cclient = chronos.connect('{}:{}'.format(settings.CHRONOS['host'], settings.CHRONOS['port']))
            if action == 'destroy':
                if request.user.has_perm("auth.can_init_job"):
                    cclient.delete(job_name)
                else:
                    raise PermissionDenied
            elif action == 'run':
                cclient.run(job_name)

            result = '{"status":"success", "msg": "%(action)s success"}'%{"action":action}
    except Exception as e:
        result = '{"status":"error", "msg": "%(action)s fail: %(error)s" }'%{"action":action, "error": html.escape(str(e))}
    return HttpResponse(result)
Пример #15
0
def dashboard(request):
    data = {}
    data['total_template'] = Template.objects.count()
    try:
        mc = MarathonClient('http://{}:{}'.format(settings.MARATHON['host'],
                                                  settings.MARATHON['port']))
        data['total_app'] = len(mc.list_apps())
    except Exception as e:
        data['total_app'] = []
    try:
        cclient = chronos.connect('{}:{}'.format(settings.CHRONOS['host'],
                                                 settings.CHRONOS['port']))
        jobs = cclient.list()
        data['total_job'] = len(cclient.list())
    except Exception as e:
        data['total_job'] = []

    data['total_watcher'] = len(settings.WATCHER_THREADS)
    return render(request, 'dashboard/dashboard.html', data)
Пример #16
0
def get_chronos_client(config, cached=False):
    """Returns a chronos client object for interacting with the API"""
    chronos_hosts = config.get_url()
    chronos_hostnames = [urlsplit(hostname).netloc for hostname in chronos_hosts]
    log.info("Attempting to connect to Chronos servers: %s" % chronos_hosts)
    if cached:
        return CachingChronosClient(
            servers=chronos_hostnames,
            username=config.get_username(),
            password=config.get_password(),
            scheduler_api_version=None,
        )
    else:
        return chronos.connect(
            servers=chronos_hostnames,
            username=config.get_username(),
            password=config.get_password(),
            scheduler_api_version=None,
        )
Пример #17
0
def new_job(request):
    data = {}
    if request.method == 'POST':
        data['msg'] = "Post"
        post_params = {}
        for key in request.POST:
            if key.startswith("filehidden"):
                fkey = key[11:]
                if(request.FILES.get(fkey, None)):
                    post_file = request.FILES[fkey]
                    file_content=""
                    for chunk in post_file.chunks():
                        file_content += chunk.decode("utf8")
                    post_params[fkey] = convert(file_content)
                else:
                    post_params[fkey] = request.POST[key]
            else:
                post_params[key] = request.POST[key]


        template = Template.objects.get(pk=post_params['template_id'])
        content = template.content%post_params
        data['content'] = content
        cclient = chronos.connect('{}:{}'.format(settings.CHRONOS['host'], settings.CHRONOS['port']))
        try:
            print(type(json.loads(content)))
            cclient.add(json.loads(content))
            data['result'] = "Success"
        except Exception as e:
            data['result'] = str(e)


    templates = Template.objects.filter(type="chronos").order_by('name').all()
    for template in templates:
        template.params = template.param_set.order_by('id')

    data['templates'] = templates
    return render(request, 'chronos_mgmt/new_job.html', data)
Пример #18
0
def new_job(request):
    data = {}
    if request.method == 'POST':
        data['msg'] = "Post"
        post_params = {}
        for key in request.POST:
            if key.startswith("filehidden"):
                fkey = key[11:]
                if (request.FILES.get(fkey, None)):
                    post_file = request.FILES[fkey]
                    file_content = ""
                    for chunk in post_file.chunks():
                        file_content += chunk.decode("utf8")
                    post_params[fkey] = convert(file_content)
                else:
                    post_params[fkey] = request.POST[key]
            else:
                post_params[key] = request.POST[key]

        template = Template.objects.get(pk=post_params['template_id'])
        content = template.content % post_params
        data['content'] = content
        cclient = chronos.connect('{}:{}'.format(settings.CHRONOS['host'],
                                                 settings.CHRONOS['port']))
        try:
            print(type(json.loads(content)))
            cclient.add(json.loads(content))
            data['result'] = "Success"
        except Exception as e:
            data['result'] = str(e)

    templates = Template.objects.filter(type="chronos").order_by('name').all()
    for template in templates:
        template.params = template.param_set.order_by('id')

    data['templates'] = templates
    return render(request, 'chronos_mgmt/new_job.html', data)
Пример #19
0
def send_to_chronos(request):
    try:
        if request.method == 'POST':
            action = request.POST.get('action', None)
            job_name = request.POST.get('name', None)
            cclient = chronos.connect('{}:{}'.format(settings.CHRONOS['host'],
                                                     settings.CHRONOS['port']))
            if action == 'destroy':
                if request.user.has_perm("auth.can_init_job"):
                    cclient.delete(job_name)
                else:
                    raise PermissionDenied
            elif action == 'run':
                cclient.run(job_name)

            result = '{"status":"success", "msg": "%(action)s success"}' % {
                "action": action
            }
    except Exception as e:
        result = '{"status":"error", "msg": "%(action)s fail: %(error)s" }' % {
            "action": action,
            "error": html.escape(str(e))
        }
    return HttpResponse(result)
Пример #20
0
    '-t', '--template',
    'template_names',
    envvar="{0}_TEMPLATES".format(CONTEXT_SETTINGS['auto_envvar_prefix']),
    type=str,
    required=True,
    help="Path of the template to use for deployment.",
    multiple=True,
)

chronos_client = click.option(
    '--chronos_url',
    'chronos_client',
    envvar="{0}_CHRONOS_URL".format(CONTEXT_SETTINGS['auto_envvar_prefix']),
    required=True,
    help='URL of the Chronos endpoint to use',
    callback=lambda c, p, v: chronos.connect([v]),
)

job_name = click.option(
    '-j', '--job-name',
    type=str,
    help='Restrict command to specific job.',
)


marathon_lb_url = click.option(
    '--marathon_lb_url',
    'marathon_lb_url',
    envvar="{0}_MARATHON_LB_URL".format(CONTEXT_SETTINGS['auto_envvar_prefix']),
    required=True,
    help="url of marathon lb",
Пример #21
0
 def __init__(self, endpoints, proto="http"):
     self.client = chronos.connect(endpoints, proto="https")
Пример #22
0
def setup_chronos_client():
    connection_string = get_service_connection_string('chronos')
    return chronos.connect(connection_string, scheduler_api_version=None)
Пример #23
0
def working_chronos(context):
    """Adds a working chronos client as context.client for the purposes of
    interacting with it in the test."""
    if not hasattr(context, 'client'):
        chronos_connection_string = get_chronos_connection_string()
        context.client = chronos.connect(chronos_connection_string)
Пример #24
0
def setup_chronos_client():
    connection_string = get_service_connection_string('chronos')
    return chronos.connect(connection_string)
Пример #25
0
def setup_chronos_client():
    connection_string = get_service_connection_string('chronos')
    return chronos.connect(connection_string)
Пример #26
0
def working_chronos(context):
    """Adds a working chronos client as context.client for the purposes of
    interacting with it in the test."""
    if not hasattr(context, 'client'):
        chronos_connection_string = get_chronos_connection_string()
        context.client = chronos.connect(chronos_connection_string)
def main():
    parser = argparse.ArgumentParser(
        description="Tool for syncing Chronos jobs from local .json files")
    parser.add_argument("--hostname",
                        metavar="<host:port>",
                        required=True,
                        help="hostname and port of the Chronos instance")
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument(
        "--sync",
        metavar="/path/to/dir",
        help=
        "path to a directory containing json files describing chronos jobs. \
                       All sub-directories will be searched for files ending in .json"
    )
    group.add_argument("--list",
                       action="store_true",
                       help="list jobs on chronos")
    parser.add_argument(
        "-n",
        action="store_true",
        default=False,
        help="dry-run, don't actually push anything to chronos")
    args = parser.parse_args()

    c = chronos.connect(args.hostname)
    cjobs = c.list()

    if args.list:
        # cjobs isn't json but this still gets us the pretty
        print(json.dumps(cjobs, sort_keys=True, indent=4))
        sys.exit(0)

    if args.sync:
        jobs = {}
        retry = {'update': [], 'add': []}
        for job in cjobs:
            jobs[job["name"]] = job

        if not os.path.isdir(args.sync):
            raise Exception("%s must be a directory" % args.sync)

        job_files = find_json_files(args.sync)
        for file in job_files:
            job = read_job_file(file)
            if not job:
                print("Skipping %s" % file)
            else:
                if job['name'] in jobs:
                    if check_update(jobs, job):
                        print("Updating job %s from file %s" %
                              (job['name'], file))
                        if not args.n:
                            try:
                                c.update(job)
                            except:
                                retry['update'].append(job)
                    else:
                        print("Job %s defined in %s is up-to-date on Chronos" %
                              (job['name'], file))
                else:
                    print("Adding job %s from file %s" % (job['name'], file))
                    if not args.n:
                        try:
                            c.add(job)
                        except:
                            retry['add'].append(job)

        attempt = 0
        while (len(retry['update']) > 0
               or len(retry['add']) > 0) and attempt < 10:
            attempt += 1
            if len(retry['update']) > 0:
                job = retry['update'].pop(0)
                try:
                    print("Retry %d for job %s" % (attempt, job['name']))
                    c.update(job)
                except:
                    retry['update'].append(job)

            if len(retry['add']) > 0:
                job = retry['add'].pop(0)
                try:
                    print("Retry %d for job %s" % (attempt, job['name']))
                    c.add(job)
                except:
                    retry['add'].append(job)

        if len(retry['update']) > 0 or len(retry['add']) > 0:
            print("Failed Jobs: %s" % sorted((retry['update'] + retry['add'])))
Пример #28
0
def get_chronos_client(endpoint, proto):
    client = chronos.connect(endpoint, proto=proto)
    return client
Пример #29
0
def main():
    parser = argparse.ArgumentParser(
        description="Monitor the status of Chronos Jobs")
    parser.add_argument("--hostname",
                        metavar="<host:port>",
                        required=True,
                        help="hostname and port of the Chronos instance")
    parser.add_argument("--prefix",
                        metavar="job-prefix",
                        required=False,
                        action="append",
                        help="if set, only check jobs matching this prefix")
    parser.add_argument("--exclude",
                        metavar="job-prefix",
                        required=False,
                        action="append",
                        help="if set, exclude jobs matching this prefix")
    parser.add_argument(
        "--warn",
        metavar="#",
        default=1,
        help="warn if at least this number of jobs are currently failed")
    parser.add_argument(
        "--crit",
        metavar="#",
        default=1,
        help="critical if at least this number of jobs are currently failed")
    args = parser.parse_args()

    fails = []
    ok = []
    unknown = []

    c = chronos.connect(args.hostname)
    cjobs = c.list()

    if not isinstance(cjobs, list):
        print "UNKNOWN: error querying chronos"
        sys.exit(3)

    for job in cjobs:
        if job['disabled']:
            continue

        if isinstance(args.prefix, list):
            if not match_prefix(args.prefix, job['name']):
                continue

        if isinstance(args.exclude, list):
            if match_prefix(args.exclude, job['name']):
                continue

        if job['lastError'] > job['lastSuccess']:
            fails.append(job['name'].encode('ascii'))
        elif job['lastSuccess']:
            ok.append(job['name'].encode('ascii'))
        else:
            unknown.append(job['name'].encode('ascii'))

    if len(unknown) > 0:
        umsg = "(%d waiting for execution or with no data)" % len(unknown)
    else:
        umsg = ''

    if len(fails) == 0:
        print "OK: %d jobs succeeded on last run %s" % (len(ok), umsg)
        sys.exit(0)
    elif len(fails) >= int(args.crit):
        print "CRITICAL: %d failed jobs: %s %s" % (
            len(fails), str(fails).strip('[]'), umsg)
        sys.exit(2)
    elif len(fails) >= int(args.warn):
        print "WARNING: %d failed jobs: %s %s" % (len(fails),
                                                  str(fails).strip('[]'), umsg)
        sys.exit(1)
def main():
    parser = argparse.ArgumentParser(description="Tool for syncing Chronos jobs from local .json files")
    parser.add_argument("--hostname", metavar="<host:port>", required=True,
                        help="hostname and port of the Chronos instance")
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument("--sync", metavar="/path/to/dir",
                       help="path to a directory containing json files describing chronos jobs. \
                       All sub-directories will be searched for files ending in .json")
    group.add_argument("--list", action="store_true", help="list jobs on chronos")
    parser.add_argument("-n", action="store_true", default=False,
                        help="dry-run, don't actually push anything to chronos")
    args = parser.parse_args()

    c = chronos.connect(args.hostname)
    cjobs = c.list()

    if args.list:
        # cjobs isn't json but this still gets us the pretty
        print json.dumps(cjobs, sort_keys=True, indent=4)
        sys.exit(0)

    if args.sync:
        jobs = {}
        retry = {'update': [], 'add': []}
        for job in cjobs:
            jobs[job["name"]] = job

        if not os.path.isdir(args.sync):
            raise Exception("%s must be a directory" % args.sync)

        job_files = find_json_files(args.sync)
        for file in job_files:
            job = read_job_file(file)
            if not job:
                print "Skipping %s" % file
            else:
                if job['name'] in jobs:
                    if check_update(jobs, job):
                        print "Updating job %s from file %s" % (job['name'], file)
                        if not args.n:
                            try:
                                c.update(job)
                            except:
                                retry['update'].append(job)
                    else:
                        print "Job %s defined in %s is up-to-date on Chronos" \
                            % (job['name'], file)
                else:
                    print "Adding job %s from file %s" % (job['name'], file)
                    if not args.n:
                        try:
                            c.add(job)
                        except:
                            retry['add'].append(job)

        attempt = 0
        while (len(retry['update']) > 0 or len(retry['add']) > 0) and attempt < 10:
            attempt += 1
            if len(retry['update']) > 0:
                job = retry['update'].pop(0)
                try:
                    print "Retry %d for job %s" % (attempt, job['name'])
                    c.update(job)
                except:
                    retry['update'].append(job)

            if len(retry['add']) > 0:
                job = retry['add'].pop(0)
                try:
                    print "Retry %d for job %s" % (attempt, job['name'])
                    c.add(job)
                except:
                    retry['add'].append(job)

        if len(retry['update']) > 0 or len(retry['add']) > 0:
            print "Failed Jobs: %s" % sorted((retry['update'] + retry['add']))