Пример #1
0
    def handle(self, *args, **options):
        verbose = int(options.get('verbosity'))
        interval = options.get('interval')
        nstreams = options.get('nstreams')
        types = options.get('types')
        flush = options.get('flush')
        demo = options.get('demo')
        span = options.get('span')
        norealtime = options.get('norealtime')

        if nstreams is None and types is None and not demo and flush:
            datastream.delete_streams()
            return
        elif flush:
            raise base.CommandError(
                "Do you really want to remove all datastream data from the database? Use only '--flush' parameter."
            )

        if nstreams is None and types is None and not flush and demo:
            types = DEMO_TYPE
            if span == '':
                span = DEMO_SPAN
        elif demo:
            raise base.CommandError("In demo mode other parameters are fixed.")

        if nstreams is None and types is None:
            nstreams = DEFAULT_NSTREAMS

        if types and check_types.match(types):
            types = split_types.findall(types)
            if nstreams is not None and len(types) != nstreams:
                raise base.CommandError(
                    "Number of stream types does not mach number of streams.")

            nstreams = len(types)

        elif types:
            raise base.CommandError(
                "Invalid stream types string. Must be a comma separated list of <int|float|enum>[(start,end)|(enum values)]."
            )

        streams = []
        for i in range(nstreams):
            if types is not None:
                typ = types[i]
            else:
                typ = ('int', '')

            if typ[0] == 'enum':
                value_type = 'nominal'
                downsamplers = ['count']
            elif typ[0] == 'graph':
                value_type = 'graph'
                downsamplers = ['count']
            else:
                value_type = 'numeric'
                downsamplers = datastream.backend.value_downsamplers

            visualization_value_downsamplers = []
            for downsampler in ['mean', 'min', 'max']:
                if downsampler in downsamplers:
                    visualization_value_downsamplers.append(downsampler)

            type_constructor, random_function, default_domain = TYPES[typ[0]]
            domain = typ[1] or default_domain
            domain_range = [type_constructor(d) for d in domain.split(',')]

            stream_id = datastream.ensure_stream(
                {'title': 'Stream %d' % i},
                {
                    'description': lorem_ipsum.paragraph(),
                    'unit_description': 'random, domain: %s' % domain,
                    'stream_number': i,
                    'visualization': {
                        'type':
                        'state' if typ is 'enum' else 'line',
                        'hidden':
                        True if typ is 'graph' else False,
                        'value_downsamplers':
                        visualization_value_downsamplers,
                        'time_downsamplers': ['mean'],
                        'minimum':
                        domain_range[0] if value_type == 'numeric' else None,
                        'maximum':
                        domain_range[1] if value_type == 'numeric' else None,
                    },
                },
                downsamplers,
                datastream.Granularity.Seconds,
                value_type=value_type,
            )

            streams.append((stream_id, typ))

        span = span.split(' ')
        if len(span) == 1 and span[0]:
            span = span[0]
            for val, key in (('days', 'd'), ('hours', 'h'), ('minutes', 'm'),
                             ('seconds', 's')):
                if span[-1] == key:
                    try:
                        s = int(span[:-1])
                    except ValueError:
                        raise base.CommandError(
                            "Time span value must be an integer.")

                    span_to = datetime.datetime.now(pytz.utc)
                    last_timestamp = self.last_timestamp(streams)

                    span_from = max(
                        span_to - datetime.timedelta(**{val: s}),
                        last_timestamp + datetime.timedelta(seconds=interval))

                    break
            else:
                raise base.CommandError("Unknown time span unit '%s'." %
                                        span[-1])

        elif len(span) == 2:
            try:
                # TODO: Support also timezone in the datetime format
                span_from, span_to = [
                    datetime.datetime.strptime(x, '%Y-%m-%dT%H:%M:%S')
                    for x in span
                ]
            except ValueError:
                raise base.CommandError(
                    "Use time format 'yyyy-mm-ddThh:mm:ss' (i.e. '2007-03-04T21:08:12')."
                )

        else:
            raise base.CommandError(
                "Invalid time span parameter. It should be one or two space-delimited values."
            )

        if span_from is not None and span_to is not None and span_from <= span_to:
            if verbose > 1:
                td = span_to - span_from
                self.stdout.write(
                    "Appending %d values from %s to %s.\n" %
                    (((td.seconds + td.days * 24 * 3600) // interval *
                      len(streams)), span_from, span_to))

            while span_from <= span_to:
                for stream_id, (typ, domain) in streams:
                    type_constructor, random_function, default_domain = TYPES[
                        typ]
                    value = random_function(*[
                        type_constructor(d)
                        for d in (domain or default_domain).split(',')
                    ])
                    datastream.append(stream_id, value, span_from)

                span_from += datetime.timedelta(seconds=interval)

            if verbose > 1:
                self.stdout.write("Done. Downsampling.\n")

            datastream.downsample_streams(until=span_to)

        if norealtime:
            return

        if verbose > 1:
            self.stdout.write(
                "Appending real-time value(s) to stream(s) every %s seconds.\n"
                % interval)

        while True:
            for stream_id, (typ, domain) in streams:
                type_constructor, random_function, default_domain = TYPES[typ]
                value = random_function(*[
                    type_constructor(d)
                    for d in (domain or default_domain).split(',')
                ])

                if verbose > 1:
                    self.stdout.write(
                        "Appending value '%s' to stream '%s'.\n" %
                        (value, stream_id))

                datastream.append(stream_id, value)

            datastream.downsample_streams()

            time.sleep(interval)
Пример #2
0
    def handle(self, *args, **options):
        verbose = int(options.get('verbosity'))
        interval = options.get('interval')
        nstreams = options.get('nstreams')
        types = options.get('types')
        flush = options.get('flush')
        demo = options.get('demo')
        span = options.get('span')

        if nstreams is None and types is None and not demo and flush:
            datastream.delete_streams()
            return
        elif flush:
            raise base.CommandError("Do you really want to remove all datastream data from the database? Use only '--flush' parameter.")

        if nstreams is None and types is None and not flush and demo:
            types = DEMO_TYPE
            if span == '':
                span = DEMO_SPAN
        elif demo:
            raise base.CommandError("In demo mode other parameters are fixed.")

        if nstreams is None and types is None:
            nstreams = DEFAULT_NSTREAMS

        if types and check_types.match(types):
            types = split_types.findall(types)
            if nstreams is not None and len(types) != nstreams:
                raise base.CommandError("Number of stream types does not mach number of streams.")

            nstreams = len(types)

        elif types:
            raise base.CommandError("Invalid stream types string. Must be a comma separated list of <int|float|enum>[(start,end)|(enum values)].")

        streams = []
        for i in range(nstreams):
            if types is None or types[i][0] != 'enum':
                downsamplers = datastream.backend.value_downsamplers
            else:
                downsamplers = []

            stream_id = datastream.ensure_stream(
                ({'name': 'stream_%d' % i},),
                (
                    'foobar',
                    {'stream_number': i},
                    {'description': lorem_ipsum.paragraph()},
                ),
                downsamplers,
                datastream.Granularity.Seconds,
            )

            if types is not None:
                streams.append((stream_id, types[i]))
            else:
                streams.append((stream_id, ('int', '')))

        span = span.split(' ')
        if len(span) == 1:
            span = span[0]
            for val, key in (('days', 'd'), ('hours', 'h')):
                if span[-1] == key:
                    try:
                        s = int(span[:-1])
                    except ValueError:
                        raise base.CommandError("Time span value must be an integer.")

                    span_to =  datetime.datetime.now(pytz.utc)
                    last_timestamp = self.last_timestamp(streams)
                    span_from = max(
                        span_to - datetime.timedelta(**{val: s}),
                        last_timestamp + datetime.timedelta(seconds=interval)
                    )

                    break
            else:
                raise base.CommandError("Unknown time span unit '%s'." % span[-1])

        elif len(span) == 2:
            try:
                # TODO: Support also timezone in the datetime format
                span_from, span_to = map(lambda x: datetime.datetime.strptime(x, '%Y-%m-%dT%H:%M:%S'), span)
            except ValueError:
                raise base.CommandError("Use time format 'yyyy-mm-ddThh:mm:ss' (i.e. '2007-03-04T21:08:12').")

        else:
            raise base.CommandError("Invalid time span parameter. It should be one or two space-delimited values.")

        if span_from is not None and span_to is not None:
            if verbose > 1:
                td = span_to - span_from
                self.stdout.write("Appending %d values from %s to %s.\n" % (((td.seconds + td.days * 24 * 3600) // interval * len(streams)), span_from, span_to))

            while span_from <= span_to:
                for stream_id, (type, domain) in streams:
                    type_constructor, random_function, default_domain = TYPES[type]
                    value = random_function(*map(type_constructor, (domain or default_domain).split(',')))
                    datastream.append(stream_id, value, span_from)

                span_from += datetime.timedelta(seconds=interval)

            if verbose > 1:
                self.stdout.write("Downsampling.\n")

            datastream.downsample_streams(until=span_to)

        if verbose > 1:
            self.stdout.write("Appending real-time value(s) to stream(s) every %s seconds.\n" % interval)

        while True:
            for stream_id, (type, domain) in streams:
                type_constructor, random_function, default_domain = TYPES[type]
                value = random_function(*map(type_constructor, (domain or default_domain).split(',')))

                if verbose > 1:
                    self.stdout.write("Appending value '%s' to stream '%s'.\n" % (value, stream_id))

                datastream.append(stream_id, value)

            datastream.downsample_streams()

            time.sleep(interval)
Пример #3
0
    def handle(self, *args, **options):
        verbose = int(options.get('verbosity'))
        interval = options.get('interval')
        nstreams = options.get('nstreams')
        types = options.get('types')
        flush = options.get('flush')
        demo = options.get('demo')
        span = options.get('span')
        norealtime = options.get('norealtime')

        if nstreams is None and types is None and not demo and flush:
            datastream.delete_streams()
            return
        elif flush:
            raise base.CommandError("Do you really want to remove all datastream data from the database? Use only '--flush' parameter.")

        if nstreams is None and types is None and not flush and demo:
            types = DEMO_TYPE
            if span == '':
                span = DEMO_SPAN
        elif demo:
            raise base.CommandError("In demo mode other parameters are fixed.")

        if nstreams is None and types is None:
            nstreams = DEFAULT_NSTREAMS

        if types and check_types.match(types):
            types = split_types.findall(types)
            if nstreams is not None and len(types) != nstreams:
                raise base.CommandError("Number of stream types does not mach number of streams.")

            nstreams = len(types)

        elif types:
            raise base.CommandError("Invalid stream types string. Must be a comma separated list of <int|float|enum>[(start,end)|(enum values)].")

        streams = []
        for i in range(nstreams):
            if types is not None:
                typ = types[i]
            else:
                typ = ('int', '')

            if typ[0] == 'enum':
                value_type = 'nominal'
                downsamplers = ['count']
            elif typ[0] == 'graph':
                value_type = 'graph'
                downsamplers = ['count']
            else:
                value_type = 'numeric'
                downsamplers = datastream.backend.value_downsamplers

            visualization_value_downsamplers = []
            for downsampler in ['mean', 'min', 'max']:
                if downsampler in downsamplers:
                    visualization_value_downsamplers.append(downsampler)

            type_constructor, random_function, default_domain = TYPES[typ[0]]
            domain = typ[1] or default_domain
            domain_range = [type_constructor(d) for d in domain.split(',')]

            stream_id = datastream.ensure_stream(
                {'title': 'Stream %d' % i},
                {
                    'description': lorem_ipsum.paragraph(),
                    'unit_description': 'random, domain: %s' % domain,
                    'stream_number': i,
                    'visualization': {
                        'type': 'state' if typ is 'enum' else 'line',
                        'hidden': True if typ is 'graph' else False,
                        'value_downsamplers': visualization_value_downsamplers,
                        'time_downsamplers': ['mean'],
                        'minimum': domain_range[0] if value_type == 'numeric' else None,
                        'maximum': domain_range[1] if value_type == 'numeric' else None,
                    },
                },
                downsamplers,
                datastream.Granularity.Seconds,
                value_type=value_type,
            )

            streams.append((stream_id, typ))

        span = span.split(' ')
        if len(span) == 1 and span[0]:
            span = span[0]
            for val, key in (('days', 'd'), ('hours', 'h'), ('minutes', 'm'), ('seconds', 's')):
                if span[-1] == key:
                    try:
                        s = int(span[:-1])
                    except ValueError:
                        raise base.CommandError("Time span value must be an integer.")

                    span_to = datetime.datetime.now(pytz.utc)
                    last_timestamp = self.last_timestamp(streams)

                    span_from = max(
                        span_to - datetime.timedelta(**{val: s}),
                        last_timestamp + datetime.timedelta(seconds=interval)
                    )

                    break
            else:
                raise base.CommandError("Unknown time span unit '%s'." % span[-1])

        elif len(span) == 2:
            try:
                # TODO: Support also timezone in the datetime format
                span_from, span_to = [datetime.datetime.strptime(x, '%Y-%m-%dT%H:%M:%S') for x in span]
            except ValueError:
                raise base.CommandError("Use time format 'yyyy-mm-ddThh:mm:ss' (i.e. '2007-03-04T21:08:12').")

        else:
            raise base.CommandError("Invalid time span parameter. It should be one or two space-delimited values.")

        if span_from is not None and span_to is not None and span_from <= span_to:
            if verbose > 1:
                td = span_to - span_from
                self.stdout.write("Appending %d values from %s to %s.\n" % (((td.seconds + td.days * 24 * 3600) // interval * len(streams)), span_from, span_to))

            while span_from <= span_to:
                for stream_id, (typ, domain) in streams:
                    type_constructor, random_function, default_domain = TYPES[typ]
                    value = random_function(*[type_constructor(d) for d in (domain or default_domain).split(',')])
                    datastream.append(stream_id, value, span_from)

                span_from += datetime.timedelta(seconds=interval)

            if verbose > 1:
                self.stdout.write("Done. Downsampling.\n")

            datastream.downsample_streams(until=span_to)

        if norealtime:
            return

        if verbose > 1:
            self.stdout.write("Appending real-time value(s) to stream(s) every %s seconds.\n" % interval)

        while True:
            for stream_id, (typ, domain) in streams:
                type_constructor, random_function, default_domain = TYPES[typ]
                value = random_function(*[type_constructor(d) for d in (domain or default_domain).split(',')])

                if verbose > 1:
                    self.stdout.write("Appending value '%s' to stream '%s'.\n" % (value, stream_id))

                datastream.append(stream_id, value)

            datastream.downsample_streams()

            time.sleep(interval)
Пример #4
0
    def handle(self, *args, **options):
        verbose = int(options.get('verbosity'))
        interval = options.get('interval')
        nstreams = options.get('nstreams')
        types = options.get('types')
        flush = options.get('flush')
        demo = options.get('demo')
        span = options.get('span')

        if nstreams is None and types is None and not demo and flush:
            datastream.delete_streams()
            return
        elif flush:
            raise base.CommandError(
                "Do you really want to remove all datastream data from the database? Use only '--flush' parameter."
            )

        if nstreams is None and types is None and not flush and demo:
            types = DEMO_TYPE
            if span == '':
                span = DEMO_SPAN
        elif demo:
            raise base.CommandError("In demo mode other parameters are fixed.")

        if nstreams is None and types is None:
            nstreams = DEFAULT_NSTREAMS

        if types and check_types.match(types):
            types = split_types.findall(types)
            if nstreams is not None and len(types) != nstreams:
                raise base.CommandError(
                    "Number of stream types does not mach number of streams.")

            nstreams = len(types)

        elif types:
            raise base.CommandError(
                "Invalid stream types string. Must be a comma separated list of <int|float|enum>[(start,end)|(enum values)]."
            )

        streams = []
        for i in range(nstreams):
            if types is None or types[i][0] != 'enum':
                downsamplers = datastream.backend.value_downsamplers
            else:
                downsamplers = []

            stream_id = datastream.ensure_stream(
                ({
                    'name': 'stream_%d' % i
                }, ),
                (
                    'foobar',
                    {
                        'stream_number': i
                    },
                    {
                        'description': lorem_ipsum.paragraph()
                    },
                ),
                downsamplers,
                datastream.Granularity.Seconds,
            )

            if types is not None:
                streams.append((stream_id, types[i]))
            else:
                streams.append((stream_id, ('int', '')))

        span = span.split(' ')
        if len(span) == 1:
            span = span[0]
            for val, key in (('days', 'd'), ('hours', 'h')):
                if span[-1] == key:
                    try:
                        s = int(span[:-1])
                    except ValueError:
                        raise base.CommandError(
                            "Time span value must be an integer.")

                    span_to = datetime.datetime.now(pytz.utc)
                    last_timestamp = self.last_timestamp(streams)
                    span_from = max(
                        span_to - datetime.timedelta(**{val: s}),
                        last_timestamp + datetime.timedelta(seconds=interval))

                    break
            else:
                raise base.CommandError("Unknown time span unit '%s'." %
                                        span[-1])

        elif len(span) == 2:
            try:
                # TODO: Support also timezone in the datetime format
                span_from, span_to = map(
                    lambda x: datetime.datetime.strptime(
                        x, '%Y-%m-%dT%H:%M:%S'), span)
            except ValueError:
                raise base.CommandError(
                    "Use time format 'yyyy-mm-ddThh:mm:ss' (i.e. '2007-03-04T21:08:12')."
                )

        else:
            raise base.CommandError(
                "Invalid time span parameter. It should be one or two space-delimited values."
            )

        if span_from is not None and span_to is not None:
            if verbose > 1:
                td = span_to - span_from
                self.stdout.write(
                    "Appending %d values from %s to %s.\n" %
                    (((td.seconds + td.days * 24 * 3600) // interval *
                      len(streams)), span_from, span_to))

            while span_from <= span_to:
                for stream_id, (type, domain) in streams:
                    type_constructor, random_function, default_domain = TYPES[
                        type]
                    value = random_function(*map(type_constructor, (
                        domain or default_domain).split(',')))
                    datastream.append(stream_id, value, span_from)

                span_from += datetime.timedelta(seconds=interval)

            if verbose > 1:
                self.stdout.write("Downsampling.\n")

            datastream.downsample_streams(until=span_to)

        if verbose > 1:
            self.stdout.write(
                "Appending real-time value(s) to stream(s) every %s seconds.\n"
                % interval)

        while True:
            for stream_id, (type, domain) in streams:
                type_constructor, random_function, default_domain = TYPES[type]
                value = random_function(*map(type_constructor, (
                    domain or default_domain).split(',')))

                if verbose > 1:
                    self.stdout.write(
                        "Appending value '%s' to stream '%s'.\n" %
                        (value, stream_id))

                datastream.append(stream_id, value)

            datastream.downsample_streams()

            time.sleep(interval)