示例#1
0
    def get(self):
        stats = self.library.getStats()
        stats['last_updated'] = utils.utc_to_local(
            stats['last_updated']).strftime("%Y-%m-%d %H:%M:%S")
        stats['created'] = utils.utc_to_local(
            stats['created']).strftime("%Y-%m-%d %H:%M:%S")

        recently_added_comics = self.library.recentlyAddedComics(10)
        recently_read_comics = self.library.recentlyReadComics(10)
        roles_list = [role.name for role in self.library.getRoles()]
        random_comic = self.library.randomComic()

        if random_comic is None:
            random_comic = type('fakecomic', (object, ), {
                'id': 0,
                'series': 'No Comics',
                'issue': 0
            })()

        self.render("index.html",
                    stats=stats,
                    random_comic=random_comic,
                    recently_added=list(recently_added_comics),
                    recently_read=list(recently_read_comics),
                    roles=roles_list,
                    server_time=int(
                        time.mktime(datetime.utcnow().timetuple()) * 1000),
                    api_key=self.application.config['security']['api_key'])
示例#2
0
 def validate_vote(self, data):
     """
     Checks the tweet contents, at least, two hashtags: the project's hashtag and a unique
     valid vote and checks the correct date of the tweet.
     """
     # Date
     data['published_on'] = utc_to_local(
         datetime.strptime(data['created_at'],
                           '%a %b %d %H:%M:%S +0000 %Y'))
     if data['published_on'] < self.metadata['start_date'] or \
         data['published_on'] > self.metadata['end_date']:
         self.logger.debug('Invalid published_on: %s', data['published_on'])
         raise InvalidVote()
     # Hashtags
     set_hashtags = set(
         [h['text'].lower() for h in data['entities']['hashtags']])
     if not self.project_hashtag in set_hashtags:
         self.logger.debug('project_hashtag not in hashtags')
         raise InvalidVote()
     vote = self.set_options.intersection(set_hashtags)
     if len(vote) != 1:
         self.logger.debug('Invalid hashtags: %s', vote)
         raise InvalidVote()
     else:
         return vote.pop()
def mine_candidate_tweets(candidates, start_date, end_date, count):
    """
    Mines tweets related to the given candidates in a certain date range
    and writes them to a MongoDB database

    :param candidates: Names of candidates to mine related tweets
    :type candidates: list(str)
    :param start_date: Lower bound of the date range
    :param start_date: datetime.datetime()
    :param end_date: Upper bound of the date range
    :param end_date: datetime.datetime()
    :param count: Number of tweets to search through
    :param count: int
    :return: Tuple containing the number of tweets mined
             and the number of tweets that had a location
    :rtype: tuple(int)
    """
    tweets_mined = 0
    tweets_with_location = 0
    for candidate in candidates:
        for tweet in tweepy.Cursor(api.search, q=candidate, tweet_mode='extended').items(count):

            tweets_mined += 1
            # Tweepy returns date in UTC time, need to convert to local time
            local_date = utc_to_local(tweet.created_at)

            # Tweet is in date range
            if True:
            # if start_date <= local_date <= end_date:
                # Tweet has a location tagged
                if tweet.coordinates is not None or tweet.place is not None:
                    tweets_with_location += 1

                    if tweet.coordinates is not None:
                        location = tweet.coordinates['coordinates']
                    else:
                        location = pick_random_coordinate(tweet.place.bounding_box.coordinates[0])

                    # Tweepy returns location as (longitude, latitude), need to swap
                    location.reverse()

                    # Analyzing sentiment of the Tweet
                    sentiment_return = text_to_sentiment(tweet.full_text)
                    sentiment_score = sentiment_return[0]
                    tokens = sentiment_return[1]

                    # Tweet does not contain any valid words
                    if len(tokens) == 0:
                        continue

                    tweet_object = Tweet(candidate, tweet.user.name, tweet.full_text, sentiment_score, tokens,
                                         local_date, location, tweet.favorite_count, tweet.retweet_count)

                    # Inserting Tweet data into the database
                    collection.insert_one(tweet_object.format_json())
                    print(candidate + ' tweet mined!')
                    print(tweets_mined)
                    print(tweets_with_location)

    return tweets_mined, tweets_with_location
示例#4
0
def format_datetime(value, format='%d.%m.%Y %H:%M:%S'):
    if not value:
        return '-'
    timezone = datetime.timezone(
        datetime.timedelta(hours=session['timezone']['hours'],
                           minutes=session['timezone']['minutes']),
        session['timezone']['timezone'])
    return utc_to_local(value, timezone).strftime(format)
示例#5
0
    def get(self):
        stats = self.library.getStats()
        stats['last_updated'] = utils.utc_to_local(stats['last_updated']).strftime("%Y-%m-%d %H:%M:%S")
        stats['created'] = utils.utc_to_local(stats['created']).strftime("%Y-%m-%d %H:%M:%S")

        recently_added_comics = self.library.recentlyAddedComics(10)
        recently_read_comics = self.library.recentlyReadComics(10)
        roles_list = [role.name for role in self.library.getRoles()]
        random_comic = self.library.randomComic()

        if random_comic is None:
            random_comic = type('fakecomic', (object,), 
             {'id':0, 'series':'No Comics', 'issue':0})()

        self.render("index.html", stats=stats,
                    random_comic=random_comic,
                    recently_added = list(recently_added_comics),
                    recently_read = list(recently_read_comics),
                    roles = roles_list,
                    server_time =  int(time.mktime(datetime.utcnow().timetuple()) * 1000),
                    api_key = self.application.config['security']['api_key']
                )
示例#6
0
def get_box(suffix, game_id, game, game_date, from_file=False):

    tries = 0
    while tries < 5:
        if from_file:
            #filename = "380329130.html"
            filename = "380329128.html"
            log.debug('filename ' + filename)
            fh = open("../Pages/" + filename)
            page = fh.read()
            fh.close()
        else:
            base = 'https://sports.yahoo.com'
            url = base + suffix
            page = get_page(url)

        try:
            stores = get_stores(source=page)
        except Exception:
            log.warning("Bad request...")
            tries += 1
            if tries == 5:
                break
            sleep(1)
            continue

        stats = stores['StatsStore']
        game_data = stores['GamesStore']['games'][game_id]

        cur_game = Game()
        # Thu, 29 Mar 2018 20:10:00 +0000
        cur_game.year = game_date.year
        cur_game.starttime = datetime.strptime(game['start_time'], '%a, %d %b %Y %H:%M:%S %z')
        # convert timestamp to PxT so date is accurate
        cur_game.start = utc_to_local(cur_game.starttime).date()
        cur_game.away_team = session.query(Team).filter(Team.yahoo_id==int(game['away_team_id'].replace('mlb.t.', ''))).one()
        cur_game.home_team = session.query(Team).filter(Team.yahoo_id==int(game['home_team_id'].replace('mlb.t.', ''))).one()
    
        # create any missing players
        check_players(
            players=stores['PlayersStore']['players'],
            away_team=cur_game.away_team,
            home_team=cur_game.home_team,
            lineups=game_data['lineups'],
            game_date=cur_game.start
        )

        return cur_game, game_data, stats

    raise Exception('Max retries hit')
def event_main_filter(event):
    '''
    判断事件中是否存在可能的UAC绕过行为
    '''
    record = win32evtlog.EvtRender(event, win32evtlog.EvtRenderEventXml)
    record_dict = xmltodict.parse(record)

    # 转换为本地时区
    evt_local_time = utc_to_local(
        record_dict['Event']['System']['TimeCreated']['@SystemTime'])
    record_dict['Event']['System']['TimeCreated'][
        '@SystemTime'] = evt_local_time

    temp_data = {}
    for data in record_dict['Event']['EventData']['Data']:
        if '#text' in data:
            temp_data[data['@Name']] = data['#text']
        elif data == None or data == 'None':
            temp_data = {}
        else:
            temp_data[data['@Name']] = None
    record_dict['Event']['EventData'] = temp_data

    evt_id = int(record_dict['Event']['System']['EventID'])

    if evt_id == 1:
        image = str(record_dict['Event']['EventData']['Image'])
        parent_image = str(record_dict['Event']['EventData']['ParentImage'])

        if parent_image == "C:\\Windows\\System32\\services.exe":
            service_outlier_executables_history[image] = 0

        if 'cmd.exe' in image:
            outlier_parents_of_cmd_history[parent_image] = 0
        # events_by_id[evt_id].append({'image': record_dict['Event']['EventData']['Image']})

        if 'ParentCommandLine' in record_dict['Event']['EventData']:
            # COM-ICMLuaUtils-bypassUAC
            # 'C:\\WINDOWS\\system32\\DllHost.exe /Processid:{3E5FC7F9-9A51-4367-9063-A120244FBEC7}':
            if '{3E5FC7F9-9A51-4367-9063-A120244FBEC7}' in \
                    record_dict['Event']['EventData']['ParentCommandLine'].upper():
                print('COM-ICMLuaUtils-bypassUAC')
                print(record_dict['Event']['EventData']['ParentCommandLine'])
                notification('COM-ICMLuaUtils-bypassUAC Detected!')

        # 判断是不是会被劫持的自动提权exe启动,是则检查对应的会被劫持的路径。
        for ex in reg_hijack_dict:
            if (image.lower().startswith(r'C:\Windows\WinSxS'.lower())
                    or image.lower().startswith(r'C:\Windows\System32'.lower())
                ) and ex.lower() in image.lower():
                # check reg symlink
                sym_path = reg_symhij_check(reg_hijack_dict[ex])
                if sym_path != None:
                    print('Possible registry UAC Hijack with symlink!')
                    print(f'Path:{sym_path}')
                    notification('Possible registry UAC Hijack with symlink!',
                                 f'Path:{sym_path}')

    # if evt_id == 2:
    #     events_by_id[evt_id].append({'image': record_dict['Event']['EventData']['Image'],
    #                                     'target name': record_dict['Event']['EventData']['TargetFilename']})
    # SYSMON EVENT ID 6 : DRIVER LOADED INTO KERNEL [DriverLoad]
    if evt_id == 6:
        if record_dict['Event']['EventData'][
                'Signature'] != 'Microsoft Windows':
            events_by_id[evt_id].append({
                'ImageLoaded':
                record_dict['Event']['EventData']['ImageLoaded'],
                'Signature':
                record_dict['Event']['EventData']['Signature']
            })
    # SYSMON EVENT ID 7 : DLL (IMAGE) LOADED BY PROCESS [ImageLoad]
    if evt_id == 7:
        if record_dict['Event']['EventData'][
                'Signature'] != 'Microsoft Windows':
            events_by_id[evt_id].append({
                'Image':
                record_dict['Event']['EventData']['Image'],
                'ImageLoaded':
                record_dict['Event']['EventData']['ImageLoaded']
            })
        # dotLocal 被劫持dll的加载
        current = events_by_id[evt_id][-1]
        if '.exe.local\\' in current['ImageLoaded'].lower():
            print("dotLocal DLL hijack detected")
            print(events_by_id[evt_id][-1])
            notification(
                "dotLocal DLL hijack detected",
                'Image: {}\nLib: {}'.format(current['Image'],
                                            current['ImageLoaded']))
    # SYSMON EVENT ID 8 : REMOTE THREAD CREATED [CreateRemoteThread]
    if evt_id == 8:
        # 远程线程注入
        events_by_id[evt_id].append({
            'SourceProcessId':
            record_dict['Event']['EventData']['SourceProcessId'],
            'SourceImage':
            record_dict['Event']['EventData']['SourceImage'],
            'TargetProcessId':
            record_dict['Event']['EventData']['TargetProcessId'],
            'TargetImage':
            record_dict['Event']['EventData']['TargetImage'],
            'StartAddress':
            record_dict['Event']['EventData']['StartAddress'],
            'StartModule':
            record_dict['Event']['EventData']['StartModule'],
            'StartFunction':
            record_dict['Event']['EventData']['StartFunction']
        })
        print("RemoteThreadCreate detected")
        print(events_by_id[evt_id][-1])
        notification(
            "RemoteThreadCreate detected", 'Source: {}\nTarget: {}'.format(
                events_by_id[evt_id][-1]['SourceImage'],
                events_by_id[evt_id][-1]['TargetImage']))

    # SYSMON EVENT ID 11 : FILE CREATED [FileCreate]
    if evt_id == 11:
        events_by_id[evt_id].append({
            'ProcessId':
            record_dict['Event']['EventData']['ProcessId'],
            'Image':
            record_dict['Event']['EventData']['Image'],
            'TargetFilename':
            record_dict['Event']['EventData']['TargetFilename']
        })
        current = events_by_id[evt_id][-1]
        # dotLocal机制的DLL劫持检测 - 判断是否有文件创建在'*.exe.local/'的路径内
        if '.exe.local\\' in current['TargetFilename'].lower():
            print("dotLocal DLL hijack file create!")
            print(events_by_id[evt_id][-1])
            notification(
                "dotLocal DLL hijack file create!",
                'Image: {}\nFile: {}'.format(current['Image'],
                                             current['TargetFilename']))

    # SYSMON EVENT ID 12 & 13 & 14 : REGISTRY MODIFICATION [RegistryEvent]
    if evt_id == 13:
        events_by_id[evt_id].append({
            'Image':
            record_dict['Event']['EventData']['Image'],
            'TargetObject':
            record_dict['Event']['EventData']['TargetObject']
        })
        current = events_by_id[evt_id][-1]
        # 打印出得到的注册表事件 - 调试用
        # print("Registry value set")
        # print(len(record_dict['Event']['EventData']['Details']))
        # print(record_dict['Event']['EventData']['Details'][:5])
        # print(type(record_dict['Event']['EventData']['Details']))
        # print(current)
        if '[Reflection.Assembly]::Load' in record_dict['Event']['EventData']['Details'] and \
                "[Microsoft.Win32.Registry]" in record_dict['Event']['EventData']['Details']:
            print("Fileless Attack - Living off the land.")
            print(current)
            notification("Fileless Attack!")

        if not record_dict['Event']['EventData']['TargetObject'].startswith(
                'HKLM'):
            # possibly HKCU
            target_path = record_dict['Event']['EventData']['TargetObject']
            target = target_path[target_path.rfind('\\') + 1:].lower()
            # 检测windir环境变量改变 - 检测部分通过windir劫持的UAC绕过方法
            if target == 'windir':
                print("Possible UACBypass: windir hijack!")
                print(current)
                notification("Possible UACBypass: windir hijack!")
            # 检测COR_ENABLE_PROFILING环境变量改变 - 检测部分通过C# profile的UAC绕过方法
            elif target.upper() == 'COR_ENABLE_PROFILING' or target.upper(
            ) == 'COR_PROFILER':
                print("Possible UACBypass: C# profile!")
                print(current)
                notification("Possible UACBypass: C# profile!")
            # print(target_path)
            value = record_dict['Event']['EventData']['Details']
            # 对每个可能被劫持路径判断一下,看看当前修改的注册表是不是用来劫持的位置
            for path in reg_hijack_dict.values():
                # print((target_path, path))
                if path in target_path:
                    print('Possible registry UAC Hijack!')
                    print(f'Path:{target_path}\nValue:{value}')
                    notification('Possible registry UAC Hijack!',
                                 f'Path:{target_path}\nValue:{value}')
def lxml():
    show_offline = request.args.get('a', 0, type=int) == 1
    lat_max = request.args.get('b', 90, type=int)
    lat_min = request.args.get('c', -90, type=int)
    lon_max = request.args.get('d', 180, type=int)
    lon_min = request.args.get('e', -180, type=int)

    if show_offline:
        observation_start = date.today()
    else:
        observation_start = datetime.utcnow() - timedelta(minutes=5)

    sq = session.query(AircraftBeacon.address,
                       func.max(AircraftBeacon.timestamp).label('lastseen')) \
        .filter(and_(between(func.ST_Y(AircraftBeacon.location_wkt), lat_min, lat_max),
                     between(func.ST_X(AircraftBeacon.location_wkt), lon_min, lon_max))) \
        .filter(AircraftBeacon.timestamp > observation_start) \
        .group_by(AircraftBeacon.address) \
        .subquery()

    position_query = session.query(AircraftBeacon, Device) \
        .outerjoin(Device, AircraftBeacon.address == Device.address) \
        .filter(and_(AircraftBeacon.address == sq.c.address,
                     AircraftBeacon.timestamp == sq.c.lastseen))

    lines = list()
    lines.append('<?xml version="1.0" encoding="UTF-8"?>')
    lines.append('<markers>')

    for [aircraft_beacon, device] in position_query.all():
        if device and (not device.tracked or not device.identified):
            continue

        code = encode(aircraft_beacon.address)

        if device is None:
            competition = ('_' + code[-2:]).lower()
            registration = code
            address = 0
        else:
            if not device.competition:
                competition = device.registration[-2:]
            else:
                competition = device.competition

            if not device.registration:
                registration = '???'
            else:
                registration = device.registration

            address = device.address

        elapsed_time = datetime.utcnow() - aircraft_beacon.timestamp
        elapsed_seconds = int(elapsed_time.total_seconds())

        lines.append('<m a="{0:.7f},{1:.7f},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13}"/>'
                     .format(aircraft_beacon.location.latitude,
                             aircraft_beacon.location.longitude,
                             competition,
                             registration,
                             aircraft_beacon.altitude,
                             utc_to_local(aircraft_beacon.timestamp).strftime("%H:%M:%S"),
                             elapsed_seconds,
                             int(aircraft_beacon.track),
                             int(aircraft_beacon.ground_speed),
                             int(aircraft_beacon.climb_rate*10)/10,
                             aircraft_beacon.aircraft_type,
                             aircraft_beacon.receiver_name,
                             address,
                             code))

    lines.append('</markers>')
    xml = '\n'.join(lines)
    return Response(xml, mimetype='text/xml')
示例#9
0
def syncAliyun(accountModel):
    try:
        if accountModel is None:
            res = {
                'result': False,
                'message': "同步失败",
            }
            return render_json(res)
        else:
            aliyunAccount = accountModel
            account_name = str(aliyunAccount.cloud_public_key)
            account_password = str(aliyunAccount.cloud_private_key)
            region = str(aliyunAccount.cloud_region)

            # print 'account_name : %s' % account_name
            # print 'account_password : %s' % account_password
            # print 'version : %s' % version
            module = 'ecs.aliyuncs.com'
            '''
            action 对应接口的接口名,请参考产品文档上对应接口的接口名
            '''
            action = 'DescribeInstances'
            version = '2014-05-26'
            PageNumber = 1
            PageSize = 30
            params = {}
            '''调用阿里云api根据需要查询的条件进行服务器列表实例查询'''
            result = utils.getInstances(module, action, account_name,
                                        account_password, region, version,
                                        PageNumber, PageSize)
            result = json.loads(result)
            print 'aliyun'
            print result
            import sys
            reload(sys)
            sys.setdefaultencoding('utf8')
            res = result
            '''判断返回的结果中是否包含正常的结果集'''
            if 'Instances' in result and result['Instances'][
                    "Instance"] is not None:
                instanceIds = []
                '''查询数据库中已存在的服务器信息'''
                instanceIdList = aliyunInstanceInfo.objects.findAllInstanceIds(
                )
                for t in range(len(instanceIdList)):
                    instanceIds.append(
                        str(instanceIdList[t].get('instance_id')))
                # print 'instanceIdList :------ %s' % instanceIdList
                # print 'instanceIds :------ %s' % instanceIds
                myResponse = result.get('Instances')
                # 说明请求成功,可以正常获得结果集
                totalCount = result.get('TotalCount')
                instanceSet = myResponse.get('Instance')
                # print 'totalCount :%s'% totalCount
                # 单次可以取完所有数据
                createInstanceList = []
                updateInstanceList = []
                params['Version'] = version
                params['PageNumber'] = PageNumber
                params['PageSize'] = PageSize
                # '''如果返回的结果小于等于单次查询的最大条数,说明调用api结果获取的服务器信息不需要再次请求接口获取信息的信息'''
                # if totalCount is not None and totalCount <= 100:
                for i in range(len(instanceSet)):
                    '''获取查询的服务器的信息,每个属性进行解析'''
                    myinstance = instanceSet[i]
                    instanceId = str(myinstance.get('InstanceId'))
                    instanceName = str(myinstance.get('InstanceName'))
                    imageId = str(myinstance.get('ImageId'))
                    RegionId = str(myinstance.get('RegionId'))
                    zone = str(myinstance.get('ZoneId'))
                    instanceType = str(myinstance.get('InstanceType'))
                    hostname = str(myinstance.get('HostName'))
                    serial_number = str(myinstance.get('SerialNumber'))
                    status = str(myinstance.get('Status'))
                    security_group_ids = str(','.join(
                        myinstance.get('SecurityGroupIds').get(
                            'SecurityGroupId')))
                    publicIp = str(
                        myinstance.get('PublicIpAddress').get('IpAddress')[0])
                    internetMaxBandwidthIn = int(
                        myinstance.get('InternetMaxBandwidthIn'))
                    internetMaxBandwidthOut = int(
                        myinstance.get('InternetMaxBandwidthOut'))
                    internetChargeType = str(
                        myinstance.get('InternetChargeType'))
                    createdTime = myinstance.get('CreationTime')
                    expiredTime = myinstance.get('ExpiredTime')
                    createdTime = utils.utc_to_local(createdTime)
                    expiredTime = utils.utc_to_local(expiredTime)
                    VpcAttributes = str(myinstance.get('VpcAttributes'))
                    EipAddress = str(myinstance.get('EipAddress'))
                    inneripaddress = str(
                        myinstance.get('VpcAttributes').get(
                            'PrivateIpAddress').get('IpAddress')[0])
                    InstanceNetworkType = str(
                        myinstance.get('InstanceNetworkType'))
                    OperationLocks = str(myinstance.get('OperationLocks'))
                    instanceChargeType = str(
                        myinstance.get('InternetChargeType'))
                    DeviceAvailable = str(myinstance.get('DeviceAvailable'))
                    IoOptimized = str(myinstance.get('IoOptimized'))
                    Cpu = myinstance.get('Cpu')
                    Memory = myinstance.get('Memory')
                    InstanceTypeFamily = str(
                        myinstance.get('InstanceTypeFamily'))
                    SpotStrategy = str(myinstance.get('SpotStrategy'))
                    NetworkInterfaces = str(
                        myinstance.get('NetworkInterfaces'))
                    '''组装阿里云服务器实例对象'''
                    instance = aliyunInstanceInfo()
                    instance.instance_id = instanceId
                    instance.instance_name = instanceName
                    instance.description = ""
                    instance.image_id = imageId
                    instance.region_id = RegionId
                    instance.zone = zone
                    instance.cpu = Cpu
                    instance.memory = Memory
                    instance.instance_type = instanceType
                    instance.instance_type_family = InstanceTypeFamily
                    instance.hostname = hostname
                    instance.serial_number = serial_number
                    instance.status = status
                    instance.security_group_ids = security_group_ids
                    instance.public_ip_addresses = publicIp
                    instance.internet_max_bandwidth_out = internetMaxBandwidthOut
                    instance.internet_max_bandwidth_in = internetMaxBandwidthIn
                    instance.internet_charge_type = internetChargeType
                    instance.created_time = createdTime
                    instance.expired_time = expiredTime
                    instance.vpc_attributes = VpcAttributes
                    instance.eip_address = EipAddress
                    instance.inner_ip_address = inneripaddress
                    instance.instance_network_type = InstanceNetworkType
                    instance.operation_locks = OperationLocks
                    instance.instance_charge_type = instanceChargeType
                    instance.spot_strategy = SpotStrategy
                    instance.device_available = DeviceAvailable
                    instance.deployment_setId = ""
                    instance.network_interfaces = NetworkInterfaces
                    instance.io_optimized = IoOptimized
                    instance.key_pair_name = ""
                    '''判断该服务器信息是否在数据库中存在,如果存在进行更新操作,如果不存在进行创建操作'''
                    if instanceId in instanceIds:
                        updateInstanceList.append(instance)
                        # print 'update instance ---------- %s' % updateInstanceList
                    else:
                        createInstanceList.append(instance)
                    key = 'InstanceIds.' + str(i + 1)
                    params[key] = instanceId
                '''调用获取服务器实例状态列表的接口,把上面查询的所有服务器的状态都查出来,因为腾讯云不支持状态和实例一同查询,所以只能分开进行查询'''
                statusrs = utils.getInstances(module, 'DescribeInstanceStatus',
                                              account_name, account_password,
                                              region, version, PageNumber,
                                              PageSize)
                statusrs = json.loads(statusrs)
                print 'aliyun-status'
                print statusrs
                if 'InstanceStatuses' in statusrs and statusrs.get(
                        'InstanceStatuses').get('InstanceStatus') is not None:
                    statusResponse = statusrs.get('InstanceStatuses')
                    # 说明请求成功,可以正常获得结果集
                    statustotalCount = statusrs.get('TotalCount')
                    InstanceStatusSet = statusResponse.get('InstanceStatus')
                    statusdict = {}
                    '''解析服务器实例列表状态并进行更新封装'''
                    if totalCount == statustotalCount:
                        for m in range(len(InstanceStatusSet)):
                            instanceStatus = InstanceStatusSet[m]
                            instanceId = instanceStatus.get('InstanceId')
                            instanceState = instanceStatus.get('Status')
                            statusdict[instanceId] = instanceState
                        for j in range(len(createInstanceList)):
                            getinstance = createInstanceList[j]
                            finalstatus = statusdict.get(
                                getinstance.instance_id)
                            getinstance.status = finalstatus
                        for k in range(len(updateInstanceList)):
                            updateinstance = updateInstanceList[k]
                            updatefinalstatus = statusdict.get(
                                updateinstance.instance_id)
                            # print 'updatefinalstatus : ---- %s' % updatefinalstatus
                            updateObject = aliyunInstanceInfo.objects.get(
                                instance_id=updateinstance.instance_id)
                            # print 'updateObject : ---- %s' % updateObject
                            updateObject.instance_id = updateinstance.instance_id
                            updateObject.instance_name = updateinstance.instance_name
                            updateObject.description = updateinstance.description
                            updateObject.image_id = updateinstance.description
                            updateObject.region_id = updateinstance.description
                            updateObject.zone = updateinstance.description
                            updateObject.cpu = updateinstance.description
                            updateObject.memory = updateinstance.description
                            updateObject.instance_type = updateinstance.description
                            updateObject.instance_type_family = updateinstance.description
                            updateObject.hostname = updateinstance.description
                            updateObject.serial_number = updateinstance.description
                            updateObject.status = updatefinalstatus
                            updateObject.security_group_ids = updateinstance.security_group_ids
                            updateObject.public_ip_addresses = updateinstance.public_ip_addresses
                            updateObject.internet_max_bandwidth_out = updateinstance.internet_max_bandwidth_out
                            updateObject.internet_max_bandwidth_in = updateinstance.internet_max_bandwidth_in
                            updateObject.internet_charge_type = updateinstance.internet_charge_type
                            updateObject.created_time = updateinstance.created_time
                            updateObject.expired_time = updateinstance.expired_time
                            updateObject.vpc_attributes = updateinstance.vpc_attributes
                            updateObject.eip_address = updateinstance.eip_address
                            updateObject.inner_ip_address = updateinstance.inner_ip_address
                            updateObject.instance_network_type = updateinstance.instance_network_type
                            updateObject.operation_locks = updateinstance.operation_locks
                            updateObject.instance_charge_type = updateinstance.instance_charge_type
                            updateObject.spot_strategy = updateinstance.spot_strategy
                            updateObject.device_available = updateinstance.device_available
                            updateObject.deployment_setId = updateinstance.deployment_setId
                            updateObject.network_interfaces = updateinstance.network_interfaces
                            updateObject.io_optimized = updateinstance.io_optimized
                            updateObject.key_pair_name = updateinstance.key_pair_name
                            updateObject.save()
                        if createInstanceList is not None and len(
                                createInstanceList) > 0:
                            aliyunInstanceInfo.objects.bulk_create(
                                createInstanceList)
                    else:
                        '''如果实例状态的数量与实例数量不一致,说明有些实例没有正常取到状态'''
                        res = {
                            'result': False,
                            'message': u"同步服务器实例信息有误",
                        }
                        return render_json(res)
                # print 'create instance ---------- %s' % createInstanceList
                '''如果返回的结果大于单次查询的最大条数,说明调用api结果获取的服务器信息需要再次请求接口获取信息的信息,循环调用该部分代码,公共部分后面将进行优化抽取'''
                if totalCount is not None and totalCount > 100:
                    num = int(math.ceil(float(totalCount) / PageSize))
                    for i in range(num, -1, -1):
                        Offset = (i + 1) * PageSize + 1
                        params = {}
                        '''调用阿里云api根据需要查询的条件进行服务器列表实例查询'''
                        result = utils.getInstances(module, action,
                                                    account_name,
                                                    account_password, region,
                                                    version, PageNumber,
                                                    PageSize)
                        result = json.loads(result)
                        print 'aliyun'
                        print result
                        import sys
                        reload(sys)
                        sys.setdefaultencoding('utf8')
                        res = result
                        '''判断返回的结果中是否包含正常的结果集'''
                        if 'Instances' in result and result['Instances'][
                                "Instance"] is not None:
                            instanceIds = []
                            '''查询数据库中已存在的服务器信息'''
                            instanceIdList = aliyunInstanceInfo.objects.findAllInstanceIds(
                            )
                            for t in range(len(instanceIdList)):
                                instanceIds.append(
                                    str(instanceIdList[t].get('instance_id')))
                            # print 'instanceIdList :------ %s' % instanceIdList
                            # print 'instanceIds :------ %s' % instanceIds
                            myResponse = result.get('Instances')
                            # 说明请求成功,可以正常获得结果集
                            totalCount = result.get('TotalCount')
                            instanceSet = myResponse.get('Instance')
                            # print 'totalCount :%s'% totalCount
                            # 单次可以取完所有数据
                            createInstanceList = []
                            updateInstanceList = []
                            params['Version'] = version
                            params['PageNumber'] = PageNumber
                            params['PageSize'] = PageSize
                            # '''如果返回的结果小于等于单次查询的最大条数,说明调用api结果获取的服务器信息不需要再次请求接口获取信息的信息'''
                            # if totalCount is not None and totalCount <= 100:
                            for i in range(len(instanceSet)):
                                '''获取查询的服务器的信息,每个属性进行解析'''
                                myinstance = instanceSet[i]
                                instanceId = str(myinstance.get('InstanceId'))
                                instanceName = str(
                                    myinstance.get('InstanceName'))
                                imageId = str(myinstance.get('ImageId'))
                                RegionId = str(myinstance.get('RegionId'))
                                zone = str(myinstance.get('ZoneId'))
                                instanceType = str(
                                    myinstance.get('InstanceType'))
                                hostname = str(myinstance.get('HostName'))
                                serial_number = str(
                                    myinstance.get('SerialNumber'))
                                status = str(myinstance.get('Status'))
                                security_group_ids = str(','.join(
                                    myinstance.get('SecurityGroupIds').get(
                                        'SecurityGroupId')))
                                publicIp = str(
                                    myinstance.get('PublicIpAddress').get(
                                        'IpAddress')[0])
                                internetMaxBandwidthIn = int(
                                    myinstance.get('InternetMaxBandwidthIn'))
                                internetMaxBandwidthOut = int(
                                    myinstance.get('InternetMaxBandwidthOut'))
                                internetChargeType = str(
                                    myinstance.get('InternetChargeType'))
                                createdTime = myinstance.get('CreationTime')
                                expiredTime = myinstance.get('ExpiredTime')
                                createdTime = utils.utc_to_local(createdTime)
                                expiredTime = utils.utc_to_local(expiredTime)
                                VpcAttributes = str(
                                    myinstance.get('VpcAttributes'))
                                EipAddress = str(myinstance.get('EipAddress'))
                                inneripaddress = str(
                                    myinstance.get('InnerIpAddress'))
                                InstanceNetworkType = str(
                                    myinstance.get('InstanceNetworkType'))
                                OperationLocks = str(
                                    myinstance.get('OperationLocks'))
                                instanceChargeType = str(
                                    myinstance.get('InternetChargeType'))
                                DeviceAvailable = str(
                                    myinstance.get('DeviceAvailable'))
                                IoOptimized = str(
                                    myinstance.get('IoOptimized'))
                                Cpu = myinstance.get('Cpu')
                                Memory = myinstance.get('Memory')
                                InstanceTypeFamily = str(
                                    myinstance.get('InstanceTypeFamily'))
                                SpotStrategy = str(
                                    myinstance.get('SpotStrategy'))
                                NetworkInterfaces = str(
                                    myinstance.get('NetworkInterfaces'))
                                '''组装阿里云服务器实例对象'''
                                instance = aliyunInstanceInfo()
                                instance.instance_id = instanceId
                                instance.instance_name = instanceName
                                instance.description = ""
                                instance.image_id = imageId
                                instance.region_id = RegionId
                                instance.zone = zone
                                instance.cpu = Cpu
                                instance.memory = Memory
                                instance.instance_type = instanceType
                                instance.instance_type_family = InstanceTypeFamily
                                instance.hostname = hostname
                                instance.serial_number = serial_number
                                instance.status = status
                                instance.security_group_ids = security_group_ids
                                instance.public_ip_addresses = publicIp
                                instance.internet_max_bandwidth_out = internetMaxBandwidthOut
                                instance.internet_max_bandwidth_in = internetMaxBandwidthIn
                                instance.internet_charge_type = internetChargeType
                                instance.created_time = createdTime
                                instance.expired_time = expiredTime
                                instance.vpc_attributes = VpcAttributes
                                instance.eip_address = EipAddress
                                instance.inner_ip_address = inneripaddress
                                instance.instance_network_type = InstanceNetworkType
                                instance.operation_locks = OperationLocks
                                instance.instance_charge_type = instanceChargeType
                                instance.spot_strategy = SpotStrategy
                                instance.device_available = DeviceAvailable
                                instance.deployment_setId = ""
                                instance.network_interfaces = NetworkInterfaces
                                instance.io_optimized = IoOptimized
                                instance.key_pair_name = ""
                                '''判断该服务器信息是否在数据库中存在,如果存在进行更新操作,如果不存在进行创建操作'''
                                if instanceId in instanceIds:
                                    updateInstanceList.append(instance)
                                    # print 'update instance ---------- %s' % updateInstanceList
                                else:
                                    createInstanceList.append(instance)
                                key = 'InstanceIds.' + str(i + 1)
                                params[key] = instanceId
                            '''调用获取服务器实例状态列表的接口,把上面查询的所有服务器的状态都查出来,因为腾讯云不支持状态和实例一同查询,所以只能分开进行查询'''
                            statusrs = utils.getInstances(
                                module, 'DescribeInstanceStatus', account_name,
                                account_password, region, version, PageNumber,
                                PageSize)
                            statusrs = json.loads(statusrs)
                            print 'aliyun-status'
                            print statusrs
                            if 'InstanceStatuses' in statusrs and statusrs.get(
                                    'InstanceStatuses').get(
                                        'InstanceStatus') is not None:
                                statusResponse = statusrs.get(
                                    'InstanceStatuses')
                                # 说明请求成功,可以正常获得结果集
                                statustotalCount = statusrs.get('TotalCount')
                                InstanceStatusSet = statusResponse.get(
                                    'InstanceStatus')
                                statusdict = {}
                                '''解析服务器实例列表状态并进行更新封装'''
                                if totalCount == statustotalCount:
                                    for m in range(len(InstanceStatusSet)):
                                        instanceStatus = InstanceStatusSet[m]
                                        instanceId = instanceStatus.get(
                                            'InstanceId')
                                        instanceState = instanceStatus.get(
                                            'Status')
                                        statusdict[instanceId] = instanceState
                                    for j in range(len(createInstanceList)):
                                        getinstance = createInstanceList[j]
                                        finalstatus = statusdict.get(
                                            getinstance.instance_id)
                                        getinstance.status = finalstatus
                                    for k in range(len(updateInstanceList)):
                                        updateinstance = updateInstanceList[k]
                                        updatefinalstatus = statusdict.get(
                                            updateinstance.instance_id)
                                        # print 'updatefinalstatus : ---- %s' % updatefinalstatus
                                        updateObject = aliyunInstanceInfo.objects.get(
                                            instance_id=updateinstance.
                                            instance_id)
                                        # print 'updateObject : ---- %s' % updateObject
                                        updateObject.instance_id = updateinstance.instance_id
                                        updateObject.instance_name = updateinstance.instance_name
                                        updateObject.description = updateinstance.description
                                        updateObject.image_id = updateinstance.description
                                        updateObject.region_id = updateinstance.description
                                        updateObject.zone = updateinstance.description
                                        updateObject.cpu = updateinstance.description
                                        updateObject.memory = updateinstance.description
                                        updateObject.instance_type = updateinstance.description
                                        updateObject.instance_type_family = updateinstance.description
                                        updateObject.hostname = updateinstance.description
                                        updateObject.serial_number = updateinstance.description
                                        updateObject.status = updatefinalstatus
                                        updateObject.security_group_ids = updateinstance.security_group_ids
                                        updateObject.public_ip_addresses = updateinstance.public_ip_addresses
                                        updateObject.internet_max_bandwidth_out = updateinstance.internet_max_bandwidth_out
                                        updateObject.internet_max_bandwidth_in = updateinstance.internet_max_bandwidth_in
                                        updateObject.internet_charge_type = updateinstance.internet_charge_type
                                        updateObject.created_time = updateinstance.created_time
                                        updateObject.expired_time = updateinstance.expired_time
                                        updateObject.vpc_attributes = updateinstance.vpc_attributes
                                        updateObject.eip_address = updateinstance.eip_address
                                        updateObject.inner_ip_address = updateinstance.inner_ip_address
                                        updateObject.instance_network_type = updateinstance.instance_network_type
                                        updateObject.operation_locks = updateinstance.operation_locks
                                        updateObject.instance_charge_type = updateinstance.instance_charge_type
                                        updateObject.spot_strategy = updateinstance.spot_strategy
                                        updateObject.device_available = updateinstance.device_available
                                        updateObject.deployment_setId = updateinstance.deployment_setId
                                        updateObject.network_interfaces = updateinstance.network_interfaces
                                        updateObject.io_optimized = updateinstance.io_optimized
                                        updateObject.key_pair_name = updateinstance.key_pair_name
                                        updateObject.save()
                                    if createInstanceList is not None and len(
                                            createInstanceList) > 0:
                                        aliyunInstanceInfo.objects.bulk_create(
                                            createInstanceList)
                                else:
                                    '''如果实例状态的数量与实例数量不一致,说明有些实例没有正常取到状态'''
                                    res = {
                                        'result': False,
                                        'message': u"同步服务器实例信息有误",
                                    }
                                    return render_json(res)
                responseResult = {
                    'result': True,
                    "content": {},
                    "message": u"操作成功"
                }
                return render_json(responseResult)

            else:
                '''无法获取阿里云服务器正常数据,返回错误结果'''
                res = {
                    'result': False,
                    'message': u"同步服务器实例信息有误",
                }
                return render_json(res)
    except Exception, e:
        traceback.print_exc()
        logger.error('syncQcloud is error :', e)
        res = {
            'result': False,
            'message': u"同步服务器实例信息有误",
        }
        return render_json(res)
示例#10
0
def should_run(api, timezone, cfg, tmp):
    return 'last_run' not in cfg or utc_to_local(
        cfg['last_run'], timezone).date() != datetime.now(timezone).date()