Exemplo n.º 1
0
def call_subprocess(cmd, stdin_data=None, stdin_async=True):
    """call sub process async
        Args:
            cmd: str, commands
            stdin_data: str, data for standard in
            stdin_async: bool, whether use async for stdin
    """
    stdin = Subprocess.STREAM if stdin_async else subprocess.PIPE
    sub_process = Subprocess(
        shlex.split(cmd),
        stdin=stdin,
        stdout=Subprocess.STREAM,
        stderr=Subprocess.STREAM,
    )

    if stdin_data:
        if stdin_async:
            yield Task(sub_process.stdin.write, stdin_data)
        else:
            sub_process.stdin.write(stdin_data)

    if stdin_async or stdin_data:
        sub_process.stdin.close()

    result, error = yield [
        Task(sub_process.stdout.read_until_close),
        Task(sub_process.stderr.read_until_close),
    ]

    raise Return((result, error))
Exemplo n.º 2
0
    def _call_ipptool(self, request):
        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
            temp_file.write(bytes(request, encoding='utf-8'))
        from tornado.process import Subprocess
        process = Subprocess([
            self.config['ipptool_path'], self.authenticated_uri, '-X',
            temp_file.name
        ],
                             stdin=subprocess.PIPE,
                             stdout=Subprocess.STREAM,
                             stderr=Subprocess.STREAM,
                             io_loop=self.io_loop)
        future = []
        self.io_loop.add_timeout(
            self.io_loop.time() + self.config['timeout'],
            functools.partial(self.timeout_handler, process.proc, future))
        try:
            stdout, stderr = yield [
                Task(process.stdout.read_until_close),
                Task(process.stderr.read_until_close)
            ]
            if future:
                raise TimeoutError
        finally:
            os.unlink(temp_file.name)

        result = plistlib.readPlistFromString(stdout)
        try:
            raise Return(result['Tests'][0])
        except (IndexError, KeyError):
            logger = logging.getLogger(__name__)
            logger.error('ipptool command failed: {} {}'.format(
                stdout, stderr))
            raise
Exemplo n.º 3
0
    def _validate_request_code(self, code, client_id, callback):
        '''
        _validate_request_code - internal method for verifying the the given nonce.
        also removes the nonce from the data_store, as they are intended for
        one-time use.
        '''
        nonce = yield Task(self.data_store.fetch, 'nonce_codes', code=code)
        if not nonce:
            raise Proauth2Error('access_denied',
                                'invalid request code: %s' % code)
        if client_id != nonce['client_id']:
            raise Proauth2Error('access_denied',
                                'invalid request code: %s' % code)
        user_id = nonce['user_id']
        expires = nonce['expires']
        yield Task(self.data_store.remove,
                   'nonce_codes',
                   code=code,
                   client_id=client_id,
                   user_id=user_id)

        if time() > expires:
            raise Proauth2Error('access_denied',
                                'request code %s expired' % code)

        callback(user_id)
Exemplo n.º 4
0
def call_subprocess(cmd, stdin_data=None, stdin_async=False):
    """
    Wrapper around subprocess call using Tornado's Subprocess class.
    """
    stdin = STREAM if stdin_async else subprocess.PIPE

    sub_process = tornado.process.Subprocess(
        cmd, stdin=stdin, stdout=STREAM, stderr=STREAM
    )

    if stdin_data:
        if stdin_async:
            yield Task(sub_process.stdin.write, stdin_data)
        else:
            sub_process.stdin.write(stdin_data)

    if stdin_async or stdin_data:
        sub_process.stdin.close()

    result, error = yield [
        Task(sub_process.stdout.read_until_close),
        Task(sub_process.stderr.read_until_close)
    ]

    raise Return((result, error))
def runCommandBackground(cmdlist, shell=False):
    """
    Wrapper around subprocess call using Tornado's Subprocess class.
    This routine can fork a process in the background without blocking the
    main IOloop, the the forked process can run for a long time without
    problem
    """

    LOG = logging.getLogger('imgstorage.imgstoragenas.NasDaemon')
    LOG.debug('Executing: ' + str(cmdlist))

    # tornado.process.initialize()

    sub_process = tornado.process.Subprocess(cmdlist,
                                             stdout=STREAM,
                                             stderr=STREAM,
                                             shell=shell)

    # we need to set_exit_callback to fetch the return value
    # the function can even be empty by it must be set or the
    # sub_process.returncode will be always None

    retval = 0
    sub_process.set_exit_callback(lambda value: value)

    (result, error) = \
        (yield [Task(sub_process.stdout.read_until_close),
                Task(sub_process.stderr.read_until_close)])

    if sub_process.returncode:
        raise ActionError('Error executing %s: %s' % (cmdlist, error))

    raise Return((result.splitlines(), error))
Exemplo n.º 6
0
    def receive(self, data):
        start, length = struct.unpack('2B', data)
        print "len:", length
        data = yield Task(self.stream.read_bytes, length)
        s_acpi = ''.join(struct.unpack_from('4s', data))  # keep 0x00
        acpi_control = struct.unpack_from('B', data)[0]

        if acpi_control & 1 == 0:  # I-FRAME
            ssn, rsn = acpi.parse_i_frame(s_acpi)
            LOG.debug("ssn: {}, rsn: {}".format(ssn, rsn))
            s_asdu = ConstBitStream(bytes=data, offset=4 * 8)
            o_asdu = asdu.ASDU(s_asdu)

        elif acpi_control & 3 == 1:  # S-FRAME
            print "B"
            rsn = acpi.parse_s_frame(s_acpi)
            print rsn

        elif acpi_control & 3 == 3:  # U-FRAME
            print "A"
            if s_acpi == acpi.STARTDT_CON:
                print 'connected'

            if s_acpi == acpi.TESTFR_ACT:
                print 'ping'
                yield Task(self.send, acpi.TESTFR_CON)
        self.stream.read_bytes(2, self.receive)
Exemplo n.º 7
0
    def rpc(self, rpc_request, properties=None, callback=None):
        '''
        Publish an RPC request.  Returns a :class:`RPCResponseFuture`.

        :param rpc_request: An instance of :class:`RPCRequest`.

        '''

        callback = stack_context.wrap(callback)

        yield Task(self.ensure_connection)
        yield Task(self.ensure_rpc_queue)
        
        if not properties:
            correlation_id = str(uuid.uuid4())
            properties = pika.BasicProperties(reply_to=self.rpc_queue,
                                              correlation_id=correlation_id)
        
        logger.info('Publishing RPC request with key: %s' %
                    rpc_request.routing_key)

        self.channel.basic_publish(exchange=rpc_request.exchange,
                                   routing_key=rpc_request.routing_key,
                                   body=rpc_request.json_params,
                                   properties=properties)

        logger.info('Constructing RPC response future with cid: %s' %
                    correlation_id)
        future = RPCResponseFuture(correlation_id,
                                   timeout=rpc_request.timeout,
                                   io_loop=self.io_loop)
        self.futures[correlation_id] = future

        callback(future)
Exemplo n.º 8
0
    def test_expiration(self):
        email = '*****@*****.**'
        token = yield Task(self.moth.create_token, email=email, expire=-1)
        authed = yield Task(self.moth.auth_token, email=email, token=token)
        self.assertFalse(authed)

        token = yield Task(self.moth.create_token, email=email, expire=1)
        authed = yield Task(self.moth.auth_token, email=email, token=token)
        self.assertTrue(authed)
Exemplo n.º 9
0
    def test_retval(self):
        email = '*****@*****.**'
        want_retval = 'hope this gets returned'
        token = yield Task(self.moth.create_token,
                           email=email,
                           retval=want_retval)

        got_retval = yield Task(self.moth.auth_token, email=email, token=token)
        self.assertEqual(want_retval, got_retval)
Exemplo n.º 10
0
    def test_remove_token(self):
        email = '*****@*****.**'
        token = yield Task(self.moth.create_token, email)
        found = self.mongo.tokens.find_one(dict(email=email, token=token))
        self.assertIsNotNone(found)

        yield Task(self.moth.remove_token, email=email, token=token)
        not_found = self.mongo.tokens.find_one(dict(email=email, token=token))
        self.assertIsNone(not_found)
Exemplo n.º 11
0
 def get(self, image_id=''):
     self.set_json_output()
     download = self.get_argument('download', '')
     if download:
         response = yield Task(
             self.api_call,
             url=self.settings['API_URL'] + '/images?download={}'.format(download),
             method='GET')
         if response and response.code in [200, 201]:
             respdata = loads(response.body.decode('utf-8'))
             links = respdata['data']
             folder = self.settings['static_path'] + '/' + str(uid())
             mkdir(folder)
             for link in links:
                 info('Downloading: ' + link['url'])
                 with open(folder + '/' + link['filename'], 'wb') as f:
                     c = pycurl.Curl()
                     c.setopt(pycurl.USERAGENT, 'Mozilla/5.0 (Windows; U; Windows NT 6.1; it; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 (.NET CLR 3.5.30729)')
                     c.setopt(c.URL, link['url'])
                     c.setopt(c.WRITEDATA, f)
                     c.perform()
                     c.close()
             curpath = dirname(realpath(curdir))
             chdir(folder)
             info('Creating zip file: ' + folder + '.zip')
             with ZipFile(folder + '.zip', 'w') as myzip:
                 for link in links:
                     myzip.write(link['filename'])
             chdir(curpath)
             rmtree(folder)
             dtexec = datetime.now() + timedelta(hours=1)
             jobid = str(uid())
             self.settings['scheduler'].add_job(
                 remove_file,
                 trigger='date',
                 name='Remove file ' + folder + '.zip at ' + str(dtexec),
                 run_date=dtexec,
                 args=[self.settings['scheduler'], folder + '.zip', jobid], coalesce=True, id=jobid)
             self.set_header('Content-Type', 'application/octet-stream')
             self.set_header('Content-Disposition', 'attachment; filename=' + folder.split('/')[-1] + '.zip')
             with open(folder + '.zip', 'rb') as f:
                 self.write(f.read())
             self.finish()
         else:
             self.response(500, 'Fail to get urls to download the images.')
             return
     else:
         response = yield Task(
             self.api_call,
             url=self.settings['API_URL'] + '/images/{}'.format(image_id),
             method='GET')
         self.set_status(response.code)
         if response.code in [200, 201]:
             self.finish(response.body)
         else:
             self.finish({'status': 'error', 'message': 'Fail to get images data.'})
Exemplo n.º 12
0
 def run_process(self):
     self.pipe = Subprocess(shlex.split(self.cmd),
                            stdout=Subprocess.STREAM,
                            stderr=Subprocess.STREAM)
     self.start = time.time()
     (out, err) = (yield [
         Task(self.pipe.stdout.read_until_close),
         Task(self.pipe.stderr.read_until_close)
     ])
     return (out, err)
Exemplo n.º 13
0
 def main():
     f = ReadOnlyFileStream('./README.md')
     first = yield Task(f.read_until, "\n")
     deuz = yield Task(f.read_until, "\n")
     yield Task(f.seek, 0)
     refirst = yield Task(f.read_until, "\n")
     f.close()
     print first, refirst
     assert first == refirst
     print first, deuz
Exemplo n.º 14
0
 def _insertChangeLog(self, branch, **kw):
     the_dict = dict(option_act=branch,
                     detail=convSimpleJson(kw))
     the_dict['time'] = '%s' %int(time.time())
     yield Task(self.db.insert, self.forms['changelog']['insert'], **the_dict)
     try:
         yield Task(self.db.insert, self.forms['changelog']['insert'], **the_dict)
         message = 'record option successed.'
     except: 
         message = 'Changelog writing failed.'
     raise Return(message)
Exemplo n.º 15
0
 def command(self, cmd, cmd_text):
     assert type(cmd_text) == str
     assert self.state == self.STATE_CONNECTED
     assert self._incommand, "overlapped commands not supported"
     assert self.current_resultset, "overlapped commands not supported"
     cmd_text = cmd_text.encode(self.charset)
     output = Stream()
     output.write_int(1, cmd)
     output.write(cmd_text)
     yield Task(self.stream.write, output.pack(0))
     print((yield Task(read_packet, self.stream)))
Exemplo n.º 16
0
 def post(self, animal_id=None, rurl=None):
     lobj = yield Task(self.get_animal_by_id, animal_id)
     if not lobj:
         self.response(404,
                       'Animal not found for the id: ' + str(animal_id))
         return
     # check data
     id_from = animal_id
     id_to = self.input_data.get('relative_id', None)
     if int(id_from) == int(id_to):
         self.response(400,
                       'Animals need to be different to define kinship.')
         return
     relation = self.input_data.get('relation', None)
     if not id_from or not id_to or not relation:
         self.response(400, 'Invalid request.')
         return
     try:
         robj = yield Task(self.get_animal_by_id, id_to)
     except Exception as e:
         info(e)
         robj = None
     if not robj:
         self.response(400, 'Relative not found with the id: %d' % (id_to))
         return
     already_relative_f = yield Task(self.check_relative, animal_id, id_to)
     already_relative_t = yield Task(self.check_relative, id_to, animal_id)
     already_relative = already_relative_f or already_relative_t
     if already_relative:
         self.response(409, 'Relation already defined.', already_relative)
         return
     valid, relation, gender = yield Task(self.relation_is_valid, lobj,
                                          robj, relation)
     if not valid:
         self.response(
             400,
             'Invalid relationship assignment request with the relation: %s. (The individual with the id %d is a "%s" animal.)'
             % (relation, int(id_to), gender))
         return
     try:
         radd = yield self.Relatives.insert({
             'id_from': int(animal_id),
             'id_to': int(id_to),
             'relation': relation.lower(),
             'created_at': datetime.now(),
             'updated_at': datetime.now()
         })
     except Exception as e:
         info(e)
         radd = None
     if radd:
         self.response(201, 'Relation added.')
     else:
         self.response(500, 'Fail to add relation.')
Exemplo n.º 17
0
    def send_users_activity(self):
        """ Publish current users in redis channel

        """
        users = yield Task(self.application.client.lrange, 'users', 0, -1)
        if users:
            try:
                users = [tornado.escape.json_decode(user) for user in users]
                yield Task(self.application.client.publish, 'users',
                           tornado.escape.json_encode(users))
            except tornado.websocket.WebSocketClosedError:
                logging.warning("Websocket closed when sending message")
Exemplo n.º 18
0
    def test_can_get_limit_usage(self):
        url = 'http://globo.com'
        key = 'limit-for-%s' % url
        self.cache.redis.delete(key)

        yield Task(self.cache.redis.zadd, key, {'a': 1, 'b': 2, 'c': 3})

        limit = yield Task(self.cache.redis.zcard, key)
        expect(limit).to_equal(3)

        limit = yield self.cache.get_limit_usage(url)
        expect(limit).to_equal(3)
Exemplo n.º 19
0
 def __FromDNSPOD(self, name, *arg, **kw):
     zid, _ = arg
     status_dict = dict(domain_name=name, status=str())
     api = self.dnspod_api['record']['list']
     f = yield self.initRequest()
     domain_id = yield Task(self._Domainid, f, name)
     origin_record_list = yield Task(f.urlPost, api, domain_id=domain_id)
     record_list = origin_record_list[1]['records'] if origin_record_list[
         0] else list()
     status = yield Task(self._InsertRecordList, record_list, zid=zid)
     status_dict['status'] = status
     raise Return(status_dict)
Exemplo n.º 20
0
 def get(self):
     self.write(self.render_string('waiting.html'))
     self.flush()
     user = self.current_user
     user_studies = yield Task(self._get_private, user)
     shared_studies = yield Task(self._get_shared, user)
     all_emails_except_current = yield Task(self._get_all_emails)
     all_emails_except_current.remove(self.current_user.id)
     self.render('private_studies.html',
                 user_studies=user_studies,
                 shared_studies=shared_studies,
                 all_emails_except_current=all_emails_except_current)
Exemplo n.º 21
0
def handle(client, stream):
    client_read = partial(read, client)

    while True:
        data = yield Task(stream.read_until, '{')
        data = data[:-1]
        count = int(data.strip()) - 1
        print('Reading {0} bytes.'.format(count))
        data = '{' + (yield Task(stream.read_bytes, count))
        client_read(data)

        if data == '{x}':
            break
Exemplo n.º 22
0
    def request_authorization(self,
                              client_id,
                              user_id,
                              response_type,
                              redirect_uri=None,
                              scope=None,
                              state=None,
                              expires=600,
                              callback=None):
        '''
        request_authorization generates a nonce, and stores it in the data_store along with the
        client_id, user_id, and expiration timestamp.
        It then returns a dictionary containing the nonce as "code," and the passed
        state.
        ---
        response_type MUST be "code." this is directly from the OAuth2 spec.
        this probably doesn't need to be checked here, but if it's in the spec I
        guess it should be verified somewhere.
        scope has not been implemented here. it will be stored, but there is no
        scope-checking built in here at this time.
        if a redirect_uri is passed, it must match the registered redirect_uri.
        again, this is per spec.
        '''
        if response_type != 'code':
            raise Proauth2Error('invalid_request',
                                'response_type must be "code"',
                                state=state)
        client = yield Task(self.data_store.fetch,
                            'applications',
                            client_id=client_id)
        if not client: raise Proauth2Error('access_denied')

        if redirect_uri and client['redirect_uri'] != redirect_uri:
            raise Proauth2Error('invalid_request', "redirect_uris don't match")

        nonce_code = self._generate_token()
        expires = time() + expires
        try:
            yield Task(self.data_store.store,
                       'nonce_codes',
                       code=nonce_code,
                       client_id=client_id,
                       expires=expires,
                       user_id=user_id,
                       scope=scope)
        except Proauth2Error as e:
            e.state = state
            raise e

        callback({'code': nonce_code, 'state': state})
Exemplo n.º 23
0
    def test_can_get_next_job_list(self):
        key = 'next-job-bucket'
        self.cache.redis.delete(key)

        for x in range(2):
            page = PageFactory.create(uuid='%d' %x, url='http://g%d.com' % x)
            yield Task(self.cache.add_next_job_bucket, page.uuid, page.url)

        data = yield Task(self.cache.get_next_job_list, 1, 10)

        expect([loads(job) for job in data]).to_equal([
            {"url":"http://g0.com","page":"0"},
            {"url":"http://g1.com","page":"1"}
        ])
Exemplo n.º 24
0
 def post(self):
     if 'new_password' in self.input_data.keys():
         if len(self.input_data['new_password']) >= 6:
             resp = self.db
             ouser = yield Task(self.get_user_by_email, self.current_user['username'])
             if ouser:
                 resp = yield Task(self.changePassword, ouser, self.input_data['new_password'])
                 self.response(resp[0], resp[1])
             else:
                 self.response(400, 'Invalid user requesting password change.')
         else:
             self.response(400, 'Password must have at least 6 characters.')
     else:
         self.response(400, 'To change your password, you must send it in a json object with the key \'new_password\'.')
Exemplo n.º 25
0
    def methods_tests(self, storage):

        res = (yield Task(storage.make_embedded,
                          project="test_project1",
                          period="month",
                          metrics=self.get_embedded_metrics()))

        self.assertTrue(isinstance(res, UUID))

        embedded_res = (yield Task(storage.get_embedded, res))
        self.assertEquals(embedded_res['project'], "test_project1")
        self.assertEquals(embedded_res['period'], "month")
        self.assertEquals(len(embedded_res['metrics']),
                          len(self.get_embedded_metrics()))
Exemplo n.º 26
0
    def receive(self, data):
        self.recived = time.time()
    
        start, length = struct.unpack('2B', data)
        #print "len:", length
        data = yield Task(self.stream.read_bytes, length)
        s_acpi = ''.join(struct.unpack_from('4s', data))  # keep 0x00
        acpi_control = struct.unpack_from('B', data)[0]

        if acpi_control & 1 == 0:  # I-FRAME
            self.ssn, self.rsn = acpi.parse_i_frame(s_acpi)
            LOG.debug("ssn: {}, rsn: {}".format(self.ssn, self.rsn))
            #s_asdu = ConstBitStream(bytes=data, offset=5*8)
            s_asdu = ConstBitStream(bytes=data, offset=4*8)
            o_asdu = asdu.ASDU(s_asdu)
            
            ###
            for o in o_asdu.objs:
                if o is asdu.MMeTf1:
                    print o.val
                if o is asdu.MSpTb1:
                    print o.val
            
            
            #self.rsn = ssn + 1
                
            #LOG.debug("send>>>>>: ssn: {}, rsn: {}".format(self.ssn, self.rsn))
            #yield Task(self.send, struct.pack('<1BHHHHH', 0x10, self.ssn, self.rsn, 0x0, 0x33, 0x16))
            #print 'send>>>>>> ' + str(ssn +1)
            #yield Task(self.sendfixed, acpi.s_frame(ssn +1))
            #print  struct.pack('<3BHHHH', 0x10, 0x01, 0x00, ssn +1, 0x0, 0x33, 0x16)
            #yield Task(self.sendraw, struct.pack('BBBHBB', 0x10, 0x01, 0x0,  0x03, 0x0, 0x16))
            yield Task(self.send, acpi.s_frame2(self.ssn + 1))
            #self.ssn += 1

        elif acpi_control & 3 == 1:  # S-FRAME
            print "S-FRAME"
            self.rsn = acpi.parse_s_frame(s_acpi)
            print self.rsn

        elif acpi_control & 3 == 3:  # U-FRAME
            print "U-FRAME"
            if s_acpi == acpi.STARTDT_CON:
                print 'connected'

            if s_acpi == acpi.TESTFR_ACT:
                print 'ping'
                yield Task(self.send, acpi.TESTFR_CON)
        self.stream.read_bytes(2, self.receive)
Exemplo n.º 27
0
    def test_increment_page_score(self):
        self.cache.redis.delete('pages-score')

        total = yield Task(self.cache.redis.zcard, 'page-scores')
        expect(int(total)).to_equal(0)

        yield self.cache.increment_page_score('page-1')

        score = yield Task(self.cache.redis.zscore, 'page-scores', 'page-1')
        expect(int(score)).to_equal(1)

        yield self.cache.increment_page_score('page-1')

        score = yield Task(self.cache.redis.zscore, 'page-scores', 'page-1')
        expect(int(score)).to_equal(2)
Exemplo n.º 28
0
    def test_add_next_job_bucket(self):
        key = 'next-job-bucket'
        self.cache.redis.delete(key)
        prefs = yield Task(self.cache.redis.get, key)
        expect(prefs).to_be_null()

        for x in range(2):
            page = PageFactory.create(uuid='%d' %x, url='http://g%d.com' % x)
            yield Task(self.cache.add_next_job_bucket, page.uuid, page.url)

        data = yield Task(self.cache.redis.zrange, key, 0, 0)
        expect(data).to_be_like([dumps({"url": "http://g0.com", "page": "0"})])

        data = yield Task(self.cache.redis.zrange, key, 1, 1)
        expect(data).to_be_like([dumps({"url": "http://g1.com", "page": "1"})])
Exemplo n.º 29
0
 def handle_stream(self, stream, address):
     while True:
         line = yield Task(stream.read_until, "\n")
         try:
             metric, timestamp, value = parse(line)
         except Exception:
             stream.close()
         serialized = struct.pack('!ff', timestamp, value)
         pipe = self.redis.pipeline()
         if metric not in self.metrics:
             pipe.sadd('metrics', metric)
             self.metrics.add(metric)
         pipe.zadd(metric, timestamp, serialized)
         pipe.publish(metric, serialized)
         yield Task(pipe.execute)
Exemplo n.º 30
0
 def test_the_whole_megillah(self):
     email = '*****@*****.**'
     ip = '19.64.2.67'
     expire = 2
     retval = 'Allan Melvin'
     token = yield Task(self.moth.create_token,
                        email=email,
                        ip=ip,
                        expire=expire,
                        retval=retval)
     authed = yield Task(self.moth.auth_token,
                         email=email,
                         token=token,
                         ip=ip)
     self.assertEqual(authed, retval)