示例#1
1
    def wait_for_slaves(self, min_slaves, timeout):
        """
        Wait the specified timeout seconds until the minimum number of slaves come online

        :param min_slaves: Minimum number of slaves to expect
        :param timeout: Max number of seconds to wait before raising an exception
        :raises: polling.TimeoutException
        :return: The count of slaves currently available
        """
        return polling.poll(
            lambda: len(self.clients.ready),
            check_success=lambda ready: ready >= min_slaves,
            timeout=timeout,
            step=1)
    def upload_transformed_data_zip(self):
        log.info('[upload] upload transform zip to molgenis')
        files = {'file': open(self.zip_src, 'rb')}
        multipart_headers = {
            'x-molgenis-token': self.config['molgenis']['token']
        }
        import_endpoint = '%s/plugin/importwizard/importFile?packageId=lifelines' % self.config[
            'molgenis']['hostname']
        res = requests.post(import_endpoint,
                            headers=multipart_headers,
                            files=files)
        res.raise_for_status()

        batch_status_endpoint = '%s%s' % (self.config['molgenis']['hostname'],
                                          res.text)

        log.debug('[upload] importing data...')
        polling.poll(lambda: requests.get(batch_status_endpoint,
                                          headers=self.json_headers).json()[
                                              'status'] != 'RUNNING',
                     step=20,
                     timeout=3600)
        res = requests.get(batch_status_endpoint, headers=self.json_headers)
        res.raise_for_status()

        res = res.json()
        batch_status = res['status']

        if batch_status != 'FINISHED':
            log.error('failed to import: %s' % json.dumps(res))
            raise Exception('failed to import:\n%s' % json.dumps(res))

        log.info('[upload] import finished with status: %s' % (batch_status))
        log.debug('batch response:\n%s' % json.dumps(res))
示例#3
0
 def run(self):
     logger.debug('Starting Hue polling')
     while run_event.is_set():
         onBeforePoll = myHue.getLight(
             whichLights[0])['json']['state']['on']
         logger.debug('Current Hue on/off state is: %s', onBeforePoll)
         logger.debug('Watching for it to change')
         # The poll times out and loops after 10 seconds so that the main thread
         # can quit this for KeyboardInterrupt etc without waiting forever
         try:
             polling.poll(lambda: myHue.getLight(whichLights[0])['json'][
                 'state']['on'] != onBeforePoll,
                          step=howOften,
                          timeout=pollerTimeout)
         except polling.TimeoutException, te:
             logger.debug('Hue poller timed out')
         onAfterPoll = myHue.getLight(whichLights[0])['json']['state']['on']
         if onAfterPoll != onBeforePoll:
             logger.debug('Lamps are now %s', ('Off', 'On')[onAfterPoll])
             logger.info('Hue event detected - Updating WeMo')
             # Now send the on/off to the WeMo switch
             switch.on() if onAfterPoll else switch.off()
             logger.debug(
                 'Turning on switch') if onAfterPoll else logger.debug(
                     'Turning off switch')
示例#4
0
def push_mfa_polling(factor_id, state_token):
    _valid = False
    url = '{org_url}/api/v1/authn/factors/{factor_id}/verify'.format(org_url=app.config['OKTA_ORG_URL'],
                                                                     factor_id=factor_id)

    payload = {
        'stateToken': state_token,
    }
    headers = {
        'Accept': 'application/json',
        'Content-Type': 'application/json',
    }

    try:
        polling.poll(
            lambda: requests.request("POST", url, headers=headers, data=json.dumps(payload)),
            check_success=push_mfa_verify,
            step=2,                         # Attempt 3 times.
            timeout=10                      # Timeout after 10 seconds.
        )
        _valid = True
    except polling.TimeoutException as te:
        while not te.values.empty():
            print(te.values.get())

    return _valid
def test_abort(find_executable, artifacts):
    """Test user's ability to abort their program using the q input

    :param fixture find_executable: Finds and returns the simulator executable
    :param fixture artifacts: Sets up the artifacts folder, organizes artifacts at teardown

    """
    TEST_PROGRAM = r"SourcePrograms/SimpleProgram.s"

    # Execute simulator
    program = xexpect.spawn("./" + find_executable, ["-f" + TEST_PROGRAM, "-d"])

    for debug_option in ('', '', '', 'q'):
        try:
            program.expect("Debug Option: ", timeout=0.2)
        except (xexpect.TIMEOUT, xexpect.EOF) as e:
            print(e)
            assert(program.isalive() == False), "Program is hung"
            break
        program.sendline(debug_option)

    try:
        program.expect("Aborting program...", timeout=0.2)
    except:
        assert(False), "Program did not abort"

    try:
        polling.poll(lambda: program.isalive() == False, step=0.1, timeout=1)
    except polling.TimeoutException as te:
        assert(False), "Program is hung"

    returncode = program.wait()
    assert(returncode == 0), "Program did not execute successfully"
示例#6
0
    def delete_all(self, timeout_secs=300):
        """Delete all locks.

        Parameters
        ----------
        timeout_secs: int
            How long to wait for internal locks (note these need to be
            cleared before external locks can be deleted)

        Raises
        ------
        APIException
        """

        try:
            polling.poll(
                lambda: len(self.get()["_embedded"]["internal_locks"]) == 0,
                step=10,
                poll_forever=False,
                timeout=timeout_secs,
            )
        except polling.TimeoutException:
            return False
        except Exception as e:
            self.client.log.error(e)
            return False

        if len(self.get()["_embedded"]["external_locks"]) > 0:
            for lock in self.get()["_embedded"]["external_locks"]:
                lock_id = lock["_links"]["self"]["href"]
                self.delete(lock_id)

        return True
示例#7
0
    def delete_all(self, timeout_secs=300):
        """Delete all locks

        Arguments:

            timeout_secs: int
                How long to wait for internal locks (note these need to be cleared before external locks can be deleted)

        Raises:

            APIException
        """

        try:
            polling.poll(
                lambda: len(self.get()["_embedded"]["internal_locks"]) == 0,
                step=60,
                poll_forever=False,
                timeout=timeout_secs,
            )
        except polling.TimeoutException:
            raise Exception("Timed out waiting for internal locks to free.")

        if len(self.get()["_embedded"]["external_locks"]) > 0:
            for lock in self.get()["_embedded"]["external_locks"]:
                lock_id = lock["_links"]["self"]["href"]
                self.delete(lock_id)

        return True
示例#8
0
    def request(self, data):
        request_id = str(uuid.uuid4())

        # Get consumers
        consumers = self.redis.smembers(KeyPattern.consumers(self.topic))

        # Set request data
        self.redis.hmset(KeyPattern.request(self.topic, request_id), data)

        # Notify each consumer
        for consumer in consumers:
            self.redis.rpush(KeyPattern.consumer(self.topic, consumer),
                             request_id)

        # Poll for responses
        def has_responses():
            for consumer in consumers:
                if not self.redis.exists(
                        KeyPattern.response(self.topic, consumer, request_id)):
                    return False

            return True

        polling.poll(has_responses, step=0.5, timeout=3)

        # Get all responses
        resps = []
        for consumer in consumers:
            ret = self.redis.hgetall(
                KeyPattern.response(self.topic, consumer, request_id))
            resps.append(ret)

        return resps
示例#9
0
 def start(self):
     Logger().info("Starting scheduler ...")
     try:
         Logger().info("Polling ...")
         poll(lambda: self.scan_commands(Configurator().db_host,
                                         Configurator().db_port,
                                         Configurator().db_name,
                                         Configurator().db_table,
                                         Configurator().db_user,
                                         Configurator().db_password),
              step=POLLING_STEP_DELAY,
              step_function=Scheduler.step_function,
              poll_forever=True)
         Logger().info("Polling is done.")
         self.shutdown()
     except TimeoutError as ex:
         Logger().exception("Polling timeout occurred. {}".format(str(ex)))
     except TimeoutException as ex:
         Logger().exception("Polling timeout occurred. {}".format(str(ex)))
     except MaxCallException as ex:
         Logger().exception("Max attempts polling error. {}".format(
             str(ex)))
     except PollingException as ex:
         Logger().exception("Polling error. {}".format(str(ex)))
     except KeyboardInterrupt:
         Logger().error("Keyboard interrupt.")
示例#10
0
    def wait_for_status(self, worker_id, status=[], timeout_secs=60):
        """
        Uses: https://github.com/justiniso/polling/blob/master/polling.py

        status: WorkerK8sStatus value, e.g. WorkerK8sStatus.configured

        raises: Exception
        """
        assert len(status) > 0, "At least one 'status' must be provided"
        assert timeout_secs >= 0, "'timeout_secs' must be >= 0"

        # current_status = self.get_k8shost(worker_id).status

        # # if we aren't configuring,  we aren't going to change state
        # if current_status.find('configuring') < 0:
        #     raise Exception('Host status is: {} - not polling for an update'.format(current_status))

        try:
            polling.poll(
                lambda: self.get_k8shost(worker_id).status in WorkerK8sStatus.status_names(status),
                step=10,
                poll_forever=False,
                timeout=timeout_secs
            )
        except polling.TimeoutException:
            status_names = [ WorkerK8sStatus.status_name(s) for s in status ]
            raise Exception(
                    "Timed out waiting for status(es): {} on K8S Worker: {}".format(
                        status_names, worker_id))
示例#11
0
 def main():
     db = FilesystemStore("/home/kiselev/db")
     poll_cooldown = 30
     locations = [Location(56.880372, 60.729744,
                           56.928178, 60.843899)]
     advertiser = PeriscopeAdvertiser(locations, db, logger)
     polling.poll(advertiser.poll, step=poll_cooldown, poll_forever=True)
def run_topic_model(job_name, num_topics):
    comprehend = boto3.client(service_name='comprehend', region_name='us-east-1')

    input_s3_url = "s3://redditdocuments/documents/" + job_name
    input_doc_format = "ONE_DOC_PER_FILE"
    output_s3_url = "s3://redditdocuments/analysis/" + job_name
    data_access_role_arn = "arn:aws:iam::449600645648:role/S3role"
    number_of_topics = num_topics

    input_data_config = {"S3Uri": input_s3_url, "InputFormat": input_doc_format}
    output_data_config = {"S3Uri": output_s3_url}

    start_topics_detection_job_result = \
        comprehend.start_topics_detection_job(NumberOfTopics=number_of_topics,
                                                                              InputDataConfig=input_data_config,
                                                                              OutputDataConfig=output_data_config,
                                                                              DataAccessRoleArn=data_access_role_arn)
    print("----Topic modelling job started----")
    try:
        poll(lambda: checkJobDone(comprehend, start_topics_detection_job_result), timeout=3000, step=1)
    except TimeoutException as tee:
        print("Value was not registered")
    print("----Topic modelling job finished----")
    status = getJobStatus(comprehend, start_topics_detection_job_result)
    output_path = status['TopicsDetectionJobProperties']['OutputDataConfig']['S3Uri'][21:]
    return output_path
示例#13
0
文件: wait.py 项目: abocz/pydriver
def wait_until(predicate, timeout_at=30, poll_every=1):
    try:
        polling.poll(predicate, poll_every, timeout=timeout_at)
    except polling.TimeoutException:
        raise PyDriverTimeoutException('Timed out after {} seconds waiting for {} to be true. '
                                       'The predicate was polled every {} seconds.'.format(timeout_at, predicate,
                                                                                           poll_every))
示例#14
0
def query_athena(query, database, output_bucket='gdot-spm-athena'):

    response = ath.start_query_execution(
        QueryString=query,
        QueryExecutionContext={
            'Database': database
        },
        ResultConfiguration={
            'OutputLocation': 's3://{}'.format(output_bucket)
        }
    )
    print ('Started query.')
    # Wait for s3 object to be created
    polling.poll(
            lambda: 'Contents' in s3.list_objects(Bucket=output_bucket, 
                                                  Prefix=response['QueryExecutionId']),
            step=0.2,
            timeout=60)
    print ('Query complete.')
    key = '{}.csv'.format(response['QueryExecutionId'])
    time.sleep(1)
    s3.download_file(Bucket=output_bucket, Key=key, Filename=key)
    df = pd.read_csv(key)
    os.remove(key)

    print ('Results downloaded.')
    return df
示例#15
0
def main():
    global do_processing
    try:
        print("Starting cleaner ...")
        print("Polling ...")

        def step_function(step):
            return step

        poll(lambda: do_cleaning(),
             step=POLLING_STEP_DELAY,
             step_function=step_function,
             poll_forever=True)
        print("Polling is done.")
        print("Stopped cleaner.")
    except TimeoutError as ex:
        print("Polling timeout occurred {0}".format(str(ex)))
    except TimeoutException as ex:
        print("Polling timeout occurred {0}".format(str(ex)))
    except MaxCallException as ex:
        do_processing = False
        print("Max attempts polling error {0}".format(str(ex)))
    except PollingException as ex:
        do_processing = False
        print("Polling error {0}".format(str(ex)))
    except KeyboardInterrupt:
        do_processing = False
        print("Keyboard interrupt.")
    except Exception as ex:
        do_processing = False
        print("Error occurred. {0}".format(str(ex)))
def test_serve_transcoded_image(hostname, large_file):
    """
    Test that the correct image content type is served after transcoding

    :param hostname: The hostname under test (this fixture is automatically injected by pytest)
    :param large_file: A large-ish filename (this fixture is automatically injected by pytest)
    """
    with open(large_file, 'r') as f:
        resp = requests.post(hostname + '/images',
                             data={'user_id': 'test-user-{}'.format(uuid.uuid4())},
                             files={'file': ('bridge.jpeg', f)})

    img_id = resp.json()['id']

    # Resize the image
    resp = requests.put(hostname + '/image/{}'.format(img_id), data={'action': 'transcode', 'extension': 'png'})
    job_id = resp.json()['job_id']

    resp = requests.get(hostname + '/image/{}'.format(img_id))
    assert resp.json()['last_job'] == job_id

    # Wait for the job to be done
    polling.poll(
        lambda: requests.get(hostname + '/image/{}'.format(img_id)),
        check_success=lambda response: response.json()['last_job_state'] == 'done',
        timeout=5,
        step=1)

    # Download the image, ensure it is smaller after resize
    download = requests.get(hostname + '/serve/{}'.format(img_id))
    assert download.headers['content-type'] == 'image/png'

    # Clean up the data
    requests.delete(hostname + '/image/{}'.format(img_id))
示例#17
0
    def modify_instance_configure_until_finished(self, mongoCap, instance, disk, flag=True):
        instance_status =None
        if flag is False:
            status, resp = mongoCap.modifyMongoDbInstance_cap(instance=instance, disk=disk, flag=flag)
            time.sleep(95)
            if status != 200 or 'error' in resp.keys():
                info_logger.debug("modify_instance_configure_until_finished interface return {}".format(resp))
                return None
            try:
                ret_code, ret_data = polling.poll(
                    lambda: mongoCap.queryFilterMongoDbs_cap(name=instance),
                    check_success=utils.polling_check_status.modify_instanceSpec_until_finished,
                    step=20, timeout=980)
            except polling.TimeoutException as e:
                info_logger.debug("create_instacne_until_finished timeout {}".format(e))
            if ret_data["mongoDbInfos"] is not None:
                instance_status = int(ret_data["mongoDbInfos"][0]["status"])
            return instance_status

        status, resp = mongoCap.modifyMongoDbInstance_cap(instance=instance,disk=disk)
        time.sleep(95)
        if status != 200 or 'error' in resp.keys():
            info_logger.debug("modify_instance_configure_until_finished interface return {}".format(resp))
            return None
        try:
            ret_code, ret_data = polling.poll(
                lambda : mongoCap.queryFilterMongoDbs_cap(name=instance),
                check_success=utils.polling_check_status.create_mongodb_until_finished,
                step=20,timeout=980)
        except polling.TimeoutException as e:
            info_logger.debug("create_instacne_until_finished timeout {}".format(e))
            return None
        if ret_data["mongoDbInfos"] is not None:
            instance_status=int(ret_data["mongoDbInfos"][0]["status"])
        return instance_status
示例#18
0
    def test_timeout_exception(self, patch_sleep, patch_time):

        # Since the timeout is < 0, the first iteration of polling should raise the error if max timeout < 0
        try:
            polling.poll(lambda: False, step=10, timeout=-1)
        except polling.TimeoutException, e:
            assert e.values.qsize() == 1, 'There should have been 1 value pushed to the queue of values'
            assert e.last is False, 'The last value was incorrect'
示例#19
0
文件: wait.py 项目: abocz/pydriver
def wait_until(predicate, timeout_at=30, poll_every=1):
    try:
        polling.poll(predicate, poll_every, timeout=timeout_at)
    except polling.TimeoutException:
        raise PyDriverTimeoutException(
            'Timed out after {} seconds waiting for {} to be true. '
            'The predicate was polled every {} seconds.'.format(
                timeout_at, predicate, poll_every))
示例#20
0
def wait_for_step_complete(emr_client, jobflow_id, step_id, sleep_interval_s):
    """
    Will poll EMR until provided step has a terminal status
    """
    poll(is_step_complete,
         args=(emr_client, jobflow_id, step_id),
         step=sleep_interval_s,
         poll_forever=True)
示例#21
0
def _wait_for_file(dir, path):
    g = os.path.join(dir, path)
    poll(
        lambda: glob.glob(g),
        timeout=_WAIT_FOR_FILE_DOWNLOAD_SECONDS,
        step=_WAIT_FOR_FILE_DOWNLOAD_POLL_SECONDS,
    )
    return glob.glob(g)
示例#22
0
文件: cli.py 项目: Noirdemort/lacr
def recv_files(arg):
    polling.poll(lambda: requests.get(f'http://{server}:{port}/checkPayload',
                                      data={
                                          'username': username
                                      }).status_code == 200,
                 check_success=respF,
                 step=5,
                 poll_forever=True)
def order_status(merch_id):
    print(merch_id)
    polling.poll(lambda: fetch_payment_details(merch_id) == 'COMPLETED' or
                 fetch_payment_details(merch_id) == 'FAILED',
                 check_success=is_correct_response,
                 step=2,
                 timeout=240)
    return client.code.get_payment_details(merch_id)
示例#24
0
def poll(step, timeout):
    polling.poll(
        lambda: pretty_print(
            requests.get("https://api.mybitx.com/api/1/ticker?pair=XBTZAR").
            json()),
        step=step,
        timeout=timeout,
    )
示例#25
0
文件: services.py 项目: 3layered/SSM
def submit(request_data, resubmit=False):
    url = conform_url_format(request_data['url'])

    if url[0:5] == 'https':
        url = url[8:]
    elif url[0:4] == 'http':
        url = url[7:]

    body = request_data['body']
    required_mem = int(request_data['memory'])
    required_cores = int(request_data['cores'])

    conn = http.client.HTTPConnection(url)
    conn.request('POST', '/ws/v1/cluster/apps/new-application')
    response = conn.getresponse().read().decode('utf-8')
    response = json.loads(response)

    submitter_id = response['application-id']
    body['application-id'] = str(submitter_id)

    maximum_resource_capacity = response['maximum-resource-capability']
    max_mem = maximum_resource_capacity['memory']
    max_core = maximum_resource_capacity['vCores']

    err_msg = {'err_msg': []}
    if required_mem > max_mem:
        err_msg['err_msg'].append(
            "YARN RM does not allow more than {} MB memory\n".format(max_mem))

    if required_cores > max_core:
        err_msg['err_msg'].append(
            "YARN RM does not allow more than {} cores\n".format(max_core))

    if len(err_msg['err_msg']) > 0:
        conn.close()
        return Response(err_msg)

    conn.request('POST',
                 '/ws/v1/cluster/apps/',
                 body=json.dumps(body),
                 headers=headers)
    response = conn.getresponse().read().decode('utf-8')

    request_data_dump = json.dumps(request_data)
    SubmitRequest(app_id=submitter_id, request_data=request_data_dump).save()

    polling.poll(lambda: wait_for_submit(url, submitter_id, request_data_dump),
                 step=3,
                 poll_forever=True)
    """    
    conn.request('PUT', '/ws/v1/cluster/apps/{}/state'.format(submitter_id), body=json.dumps({"state": "KILLED"}),
                 headers=headers)
    if conn.getresponse():
        pass
    """
    conn.close()

    return Response(response)
示例#26
0
 def polled_expected_status(hc, stack_id, status):
     try:
         poll(lambda: hc.is_status_expected(hc, stack_id, status),
              timeout=1800,
              step=10)
         return True
     except:  # noqa: E722
         print("Waiting for polling stack status Timed out")
         return False
示例#27
0
 def poll_report(self, status_url: str):
   """polls status url once a second until it gets a 200 response
   , and times out after one minute"""
   polling.poll(
       lambda: self._request.get(
           '{0}/{1}'.format(self._iq_url, status_url)).text,
       check_success=self.__handle_response,
       step=1,
       timeout=60)
示例#28
0
    def no_results(self) -> bool:

        def get_items():
            try:
                return self.browser.find_element_by_class_name("searchbar_results-empty").is_displayed()
            except NoSuchElementException:
                return False

        polling.poll(get_items, step=0.1, timeout=5)
        return True
示例#29
0
 def check_auth(self) -> bool:
     try:
         polling.poll(
             lambda: self._success_partial_url in self._driver.current_url,
             step=3,
             timeout=30,
         )
     except polling.TimeoutException:
         return False
     return True
示例#30
0
 def test_max_call_exception(self):
     """
     Test that a MaxCallException will be raised 
     """
     tries = 100
     try:
         polling.poll(lambda: False, step=0, max_tries=tries)
     except polling.MaxCallException, e:
         assert e.values.qsize() == tries, 'Poll function called the incorrect number of times'
         assert e.last is False, 'The last value was incorrect'
示例#31
0
def wait_for_runtime_status(bot_uuid: str, status: AssistantRuntimeStatus):
    settings = ToolSettings()
    current_environment = settings.get_current_environment()
    robo = get_robo_client(current_environment)
    polling.poll(
        lambda: robo.assistants.runtimes.get(bot_uuid).content.status ==
        status,
        step=5,
        poll_forever=True,
    )
示例#32
0
def wait_for_result(response):
    if response.ok:
        job_id = response.json()['id']
        print(f'Waiting for job (ID: {job_id}) to finish...')
        # check every 10 seconds until the job is finished (wait up to 48 hours)
        poll(lambda: is_done(job_id), step=10, timeout=172800)
        result = get_job_result(job_id)
        return result
    else:
        raise HTTPError(f'Failed with response : {response.json()}')
示例#33
0
def wait_for_pipeline_status_to_be_completed(build_id):
    pipeline_status_url = organization_url + "/_apis/build/Builds/" + str(
        build_id)

    polling.poll(lambda: json.loads(
        requests.request("GET", pipeline_status_url, headers=headers).text).
                 get('status'),
                 check_success=is_correct_response,
                 step=1,
                 timeout=timeout)
 def _wait_for_deletion_complete(self):
     """Poll GET request till 404 occured."""
     try:
         polling.poll(lambda: self.get().status_code == 404,
                      timeout=120,
                      step=2)
     except polling.TimeoutException as exception:
         self._exceptions.append(exception)
     else:
         self._deleted_status = True
示例#35
0
文件: cli.py 项目: Noirdemort/lacr
def recv_msgs(arg):
    if arg():
        exit(0)
    polling.poll(lambda: requests.get(
        f'http://{server}:{port}/isThereAMessage', data={
            'username': username
        }).status_code == 200,
                 check_success=resp,
                 step=2,
                 poll_forever=True)
示例#36
0
    def test_timeout_exception(self, patch_sleep, patch_time):

        # Since the timeout is < 0, the first iteration of polling should raise the error if max timeout < 0
        try:
            polling.poll(lambda: False, step=10, timeout=-1)
        except polling.TimeoutException as e:
            assert e.values.qsize() == 1, 'There should have been 1 value pushed to the queue of values'
            assert e.last is False, 'The last value was incorrect'
        else:
            assert False, 'No timeout exception raised'

        # Test happy path timeout
        val = polling.poll(lambda: True, step=0, timeout=0)
        assert val is True, 'Val was: {} != {}'.format(val, True)
示例#37
0
def wait_for_inshape_loading(driver):

	def is_element_present_and_displayed(driver, classname):

		hover_elems = driver.find_elements_by_class_name(classname)
		if not hover_elems:
			return False

		for elem in hover_elems:
			if not elem.is_displayed():
				return False

		return True

	polling.poll(
		lambda: not is_element_present_and_displayed(driver, 'hoverLoading'),
		step=1,
		timeout=90,
	)
def test_serve_cropped_image(hostname, large_file):
    """
    Test that the correct image is served after cropped

    :param hostname: The hostname under test (this fixture is automatically injected by pytest)
    :param large_file: A large-ish filename (this fixture is automatically injected by pytest)
    """
    with open(large_file, 'r') as f:
        resp = requests.post(hostname + '/images',
                             data={'user_id': 'test-user-{}'.format(uuid.uuid4())},
                             files={'file': ('bridge.jpeg', f)})

    img_id = resp.json()['id']

    # Get the original size
    download = requests.get(hostname + '/serve/{}'.format(img_id))
    original_size = download.headers['content-length']

    # Resize the image
    resp = requests.put(hostname + '/image/{}'.format(img_id), data={'action': 'crop', 'box': '25,25,900,200'})
    job_id1 = resp.json()['job_id']

    resp = requests.get(hostname + '/image/{}'.format(img_id))
    assert resp.json()['last_job'] == job_id1

    # Wait for the job to be done
    polling.poll(
        lambda: requests.get(hostname + '/image/{}'.format(img_id)),
        check_success=lambda response: response.json()['last_job_state'] == 'done',
        timeout=5,
        step=1)

    # Download the image, ensure it was actually cropped
    download = requests.get(hostname + '/serve/{}'.format(img_id))
    assert download.headers['content-length'] < original_size
    assert download.headers['content-type'] == 'image/jpeg'

    # Clean up the data
    requests.delete(hostname + '/image/{}'.format(img_id))
示例#39
0
def _wait_result(exp_prefix, exp_name, timeout):
    """
    Poll for the sync of params.pkl (currently hardcoded) from S3, indicating that the task is done.
    
    :param exp_prefix: str, experiment name prefix (dir where results are expected to be stored)
    :param exp_name: str, experiment name. Name of dir below exp_prefix where result files of individual run are
        expected to be stored
    :param timeout: int, polling timeout in seconds
    :return bool. False if the polling times out. True if successful.
    """
    result_path = os.path.join(config.LOG_DIR, "s3", exp_prefix, exp_name, 'params.pkl')
    print("Polling for results in",result_path) 
    try:
        file_handle = polling.poll(
            lambda: open(result_path),
            ignore_exceptions=(IOError,),
            timeout=timeout,
            step=60)
        file_handle.close()
    except polling.TimeoutException:
        return False
    return True
示例#40
0
def poll(action_fn, error_message, *error_message_args):
    """
    Poll until action_fn returns something truthy. After GLOBAL_TIMEOUT throw an exception.

    action_fn may return:
    - a tuple: first element is a result (truthy or falsy), second element any detail
    - any other type: truthy or falsy decides whether the polling has been successful or not

    error_message may be:
    - a string to be formatted with error_message_args
    - a callable returning such a string"""
    details = {'last_seen': None}
    def wrapped_action_fn():
        possible_result = action_fn()
        if isinstance(possible_result, tuple) and len(possible_result) == 2:
            details['last_seen'] = possible_result[1]
            return possible_result[0]
        else:
            return possible_result
    try:
        return polling.poll(
            wrapped_action_fn,
            timeout=GLOBAL_TIMEOUT,
            step=5
        )
    except polling.TimeoutException:
        if callable(error_message):
            error_message_template = error_message()
        else:
            error_message_template = error_message
        built_error_message = error_message_template % tuple(error_message_args)
        if 'last_seen' in details:
            built_error_message = built_error_message + "\n" + pformat(details['last_seen'])
            if isinstance(details['last_seen'], ConnectionError):
                host = urlparse(details['last_seen'].request.url).netloc
                built_error_message = built_error_message + ("\nHost: %s" % host)
                built_error_message = built_error_message + ("\nIp: %s" % debug.get_host_ip(host))
        raise TimeoutError.giving_up_on(built_error_message)
示例#41
0
            print "[-] Error: New instance did not assign itself EIP: {0}. Cannot proceed with judgment. " \
                  "Initiating rollback.".format(args.eip)
            rollback(deployment_id)
            assign_eip(args.eip, rbinstance_id)
            sys.exit(2)

    if not args.bypass:
        if args.nofollow:
            allow_redirects = False
        else:
            allow_redirects = True
        try:
            polling.poll(
                lambda: requests.get(test_url, allow_redirects=allow_redirects, timeout=requests_timeout),
                check_success=qa,
                ignore_exceptions=(requests.exceptions.Timeout, requests.exceptions.ConnectionError),
                step=3,
                timeout=judge_timeout
            )
        except polling.TimeoutException, te:
            print "[-] Error running test: Test url failed to meet test criteria."
            while not te.values.empty():
                print "{0}".format(te.values.get())
            print "[+] Initiating rollback of deployment."
            rollback(deployment_id)
            if args.eip:
                assign_eip(args.eip, rbinstance_id)
            sys.exit(2)

        print "[+] Tests passed. Proceeding deployment."
        proceed(deployment_id)
示例#42
0
def main():
    version = '1.3'

    parser = argparse.ArgumentParser(description="AMI Asgard Deployment Script.")
    parser.add_argument("-n", "--nofollow", action='store_true', help="Do not follow redirects for the test URL.")
    parser.add_argument("--eip", help="Elastic IP address for rollback. (EIP based deployments only).", required=False)
    parser.add_argument("-b", "--bypass", action="store_true", help="Bypass the URL test")
    parser.add_argument("-j", "--judgment", help="Judgment timeout Default: 150 seconds", required=False, const=150,
                        nargs='?', type=int, default=150)
    parser.add_argument("-s", "--startup", help="Asgard in-service startup timeout. Default: 10 minutes",
                        required=False, const=10, nargs='?', type=int, default=10)
    parser.add_argument('-t', '--timeout', help='Asgard polling timeout. (How long to wait for deployment to enter'
                                                'judgment phase before rolling back). '
                        'Default: 600 seconds', required=False,
                        const=600, nargs='?', type=int, default=600)
    parser.add_argument('asg_id')
    parser.add_argument('ami_id')
    parser.add_argument('test_url', nargs='?')
    parser.add_argument('regex', nargs='?')
    args = parser.parse_args()

    print "AMI Asgard Deployment Script {0}".format(version)

    asg_id = args.asg_id
    ami_id = args.ami_id
    poll_timeout = args.timeout
    judge_timeout = args.judgment
    startup_timeout = args.startup

    if not args.bypass:
        test_url = args.test_url
        if args.regex:
            global regex
            regex = args.regex

    print "[+] Asgard Host: {0}".format(asgard_host)
    print "[+] EC2 Region: {0}".format(ec2_region)
    print "[+] ASG: {0}".format(asg_id)
    print "[+] AMI to Launch: {0}".format(ami_id)

    if args.eip:
        print "[+] Determining rollback instance_id for: {0}".format(args.eip)
        rbinstance_id = search_ip(args.eip)
        print "{0}".format(rbinstance_id)

    query = base_url + '/deployment/prepare?id=' + asg_id
    f = urllib2.urlopen(query)
    deflcjson = f.read()
    f.close()

    deflc = json.loads(deflcjson)
    deflc['lcOptions']['imageId'] = ami_id
    deflc['deploymentOptions'] = {
        "clusterName": asg_id,
        "notificationDestination": notify,
        "steps": [
            {"type": "CreateAsg"},
            {"type": "Resize", "targetAsg": "Next", "capacity": deflc['asgOptions']['minSize'],
             "startUpTimeoutMinutes": startup_timeout},
            {"type": "DisableAsg", "targetAsg": "Previous"},
            {"type": "Judgment", "durationMinutes": 30},
            {"type": "DeleteAsg", "targetAsg": "Previous"},
        ]
    }

    post_url = base_url + '/deployment/start'
    headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
    response = requests.post(post_url, data=json.dumps(deflc), headers=headers)

    print response
    print response.text

    print "[+] Waiting for deployment to enter judgement state. Timeout is: {0}".format(poll_timeout)
    deployment_id = response.json()["deploymentId"]
    poll_url = base_url + '/task/show/' + deployment_id + '.json'

    time.sleep(15)

    try:
        polling.poll(
            lambda: requests.get(poll_url, allow_redirects=True, timeout=requests_timeout),
            check_success=judgement_ready,
            step=30,
            timeout=poll_timeout
        )
    except polling.TimeoutException, te:
        print "[-] Error waiting for judgement."
        while not te.values.empty():
            print "{0}".format(te.values.get())
        print "{0}".format(get_error(poll_url))
        sys.exit(2)
示例#43
0
    def test_arg_validation(self):
        """Tests various permutations of calling with invalid args"""

        # No function
        try:
            polling.poll()
        except TypeError:
            pass
        else:
            assert False, 'No error raised with no args'

        try:
            polling.poll(lambda: True)
        except TypeError:
            pass
        else:
            assert False, 'No error raised with no step'

        try:
            polling.poll(lambda: True, step=1)
        except AssertionError:
            pass
        else:
            assert False, 'No error raised without specifying poll_forever or a timeout/max_tries'

        try:
            polling.poll(lambda: True, step=1, timeout=1, max_tries=1, poll_forever=True)
        except AssertionError:
            pass
        else:
            assert False, 'No error raised when specifying poll_forever with timeout/max_tries'

        # Valid options
        polling.poll(lambda: True, step=1, poll_forever=True)
        polling.poll(lambda: True, step=1, timeout=1)
        polling.poll(lambda: True, step=1, max_tries=1)
        polling.poll(lambda: True, step=1, timeout=1, max_tries=1)
示例#44
0
		elem_pass.send_keys(password)
		driver.find_element_by_class_name('login-lonely-button').click()

		# Navigates to the manufacturing tool 
		driver.find_element_by_link_text("Manufacturer tool").click()
		
		wait_for_inshape_loading(driver)

		# inputs the tray name
		elem_tray = driver.find_element_by_id("searchBox")
		elem_tray.send_keys(tray_final)
		driver.find_element_by_xpath("//select[@name='searchType']/option[9]").click()
		
		cooling_tab = polling.poll(
			lambda: driver.find_element_by_id("substatus-344"),
			step=1,
			timeout=90,
			ignore_exceptions=(Exception,)
		)
		cooling_tab.click()

		wait_for_inshape_loading(driver)

		# navagates to the cooling tab
		driver.find_element_by_id("substatus-344").click()

		wait_for_inshape_loading(driver)

		# select the tray by tray name
		driver.find_element_by_id("assign-bulk").click()

		# clicks update
示例#45
0
def main():
    parser, options, arguments = parse_options()

    # setup logging
    setup_logging(options.loglevel, options.logfile)
    logger = logging.getLogger(__name__)
    
    if options.show_version:
        print "Locust %s" % (version,)
        sys.exit(0)

    if os.path.isdir(options.locustfile):
        all_locustfiles = collect_locustfiles(options.locustfile)
    else:
        locustfile = find_locustfile(options.locustfile)
        if not locustfile:
            logger.error("Could not find any locustfile! Ensure file ends in '.py' and see --help for available options.")
            sys.exit(1)

        all_locustfiles = load_locustfile(locustfile)

    logger.info("All available locustfiles: {}".format(all_locustfiles))

    # Use the first locustfile for the default locusts
    locusts = all_locustfiles.values()[0]

    if options.list_commands:
        console_logger.info("Available Locusts:")
        for name in locusts:
            console_logger.info("    " + name)
        sys.exit(0)

    if not locusts:
        logger.error("No Locust class found!")
        sys.exit(1)

    # make sure specified Locust exists
    if arguments:
        missing = set(arguments) - set(locusts.keys())
        if missing:
            logger.error("Unknown Locust(s): %s\n" % (", ".join(missing)))
            sys.exit(1)
        else:
            names = set(arguments) & set(locusts.keys())
            locust_classes = [locusts[n] for n in names]
    else:
        locust_classes = locusts.values()
    
    if options.show_task_ratio:
        console_logger.info("\n Task ratio per locust class")
        console_logger.info( "-" * 80)
        print_task_ratio(locust_classes)
        console_logger.info("\n Total task ratio")
        console_logger.info("-" * 80)
        print_task_ratio(locust_classes, total=True)
        sys.exit(0)
    if options.show_task_ratio_json:
        from json import dumps
        task_data = {
            "per_class": get_task_ratio_dict(locust_classes), 
            "total": get_task_ratio_dict(locust_classes, total=True)
        }
        console_logger.info(dumps(task_data))
        sys.exit(0)

    if options.master and options.no_web and not options.min_slaves:
        logger.error("When running --master and --no-web, you must specify --min-slaves to be available before starting to swarm")
        sys.exit(1)

    if options.master and options.no_web and not (options.timeout or options.num_requests):
        logger.error("When running --master and --no-web, you must specify either --num-request or --timeout to tell the slaves when to stop running each locustfile")
        sys.exit(1)

    if not options.no_web and not options.slave:
        # spawn web greenlet
        logger.info("Starting web monitor at %s:%s" % (options.web_host or "*", options.port))
        main_greenlet = gevent.spawn(web.start, locust_classes, options)

    if options.slave:
        logger.info("Waiting for master to become available")
        try:
            runners.locust_runner = polling.poll(
                lambda: SlaveLocustRunner(locust_classes, options, available_locustfiles=all_locustfiles),
                timeout=60,
                step=1,
                ignore_exceptions=(socket.error,))

        except polling.TimeoutException, e:
            logger.error("Failed to connect to the Locust master: %s", e.last)
            sys.exit(-1)

        main_greenlet = runners.locust_runner.greenlet
示例#46
0
#! coding: utf-8

import datetime
import polling


def print_now():
    """print_now"""
    print datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')

if __name__ == '__main__':
    # run print_now every step sec
    polling.poll(print_now, step=1, poll_forever=True)
示例#47
0
    @patch('time.sleep', return_value=None)
    @patch('time.time', return_value=0)
    def test_timeout_exception(self, patch_sleep, patch_time):

        # Since the timeout is < 0, the first iteration of polling should raise the error if max timeout < 0
        try:
            polling.poll(lambda: False, step=10, timeout=-1)
        except polling.TimeoutException, e:
            assert e.values.qsize() == 1, 'There should have been 1 value pushed to the queue of values'
            assert e.last is False, 'The last value was incorrect'
        else:
            assert False, 'No timeout exception raised'

        # Test happy path timeout
        val = polling.poll(lambda: True, step=0, timeout=0)
        assert val is True, 'Val was: {} != {}'.format(val, True)

    def test_max_call_exception(self):
        """
        Test that a MaxCallException will be raised 
        """
        tries = 100
        try:
            polling.poll(lambda: False, step=0, max_tries=tries)
        except polling.MaxCallException, e:
            assert e.values.qsize() == tries, 'Poll function called the incorrect number of times'
            assert e.last is False, 'The last value was incorrect'
        else:
            assert False, 'No MaxCallException raised'