Beispiel #1
0
    def modify_node_interface(self, node_id, roles_blade):
        log('Modify interface config for node %s' % node_id)
        interface_yaml = ('%s/node_%s/interfaces.yaml'
                          % (self.yaml_config_dir, node_id))
        check_file_exists(interface_yaml)
        backup('%s/node_%s' % (self.yaml_config_dir, node_id))

        with io.open(interface_yaml) as stream:
            interfaces = yaml.load(stream)

        net_name_id = {}
        for interface in interfaces:
            for network in interface['assigned_networks']:
                net_name_id[network['name']] = network['id']

        type = self.dea.get_node_property(roles_blade[1], 'interfaces')
        interface_config = self.dea.get_property(type)

        for interface in interfaces:
            interface['assigned_networks'] = []
            if interface['name'] in interface_config:
                for net_name in interface_config[interface['name']]:
                    net = {}
                    net['id'] = net_name_id[net_name]
                    net['name'] = net_name
                    interface['assigned_networks'].append(net)

        with io.open(interface_yaml, 'w') as stream:
            yaml.dump(interfaces, stream, default_flow_style=False)
    def set_filter_date(self):

        dialog = xbmcgui.Dialog()
        if self.start_date == '':
            self.start_date = str(datetime.datetime.now())[:10]
        if self.end_date == '':
            self.end_date = str(datetime.datetime.now())[:10]

        try:
            d = dialog.numeric(1, common.getstring(30117) ,strftime("%d/%m/%Y",strptime(self.start_date,"%Y-%m-%d")) )
            if d != '':    
                self.start_date = strftime("%Y-%m-%d",strptime(d.replace(" ","0"),"%d/%m/%Y"))
            else:
                self.start_date =''
            common.log('', str(self.start_date))
            
            d = dialog.numeric(1, common.getstring(30118) ,strftime("%d/%m/%Y",strptime(self.end_date,"%Y-%m-%d")) )
            if d != '':
                self.end_date = strftime("%Y-%m-%d",strptime(d.replace(" ","0"),"%d/%m/%Y"))
            else:
                self.end_date =''
            common.log('', str(self.end_date))
        except:
            pass

        if self.start_date != '' or self.end_date != '':
            self.getControl( BUTTON_DATE ).setLabel( self.start_date + ' ... ' + self.end_date )
        else:
            self.getControl( BUTTON_DATE ).setLabel( common.getstring(30164) )
        self.getControl( BUTTON_DATE ).setVisible(False)
        self.getControl( BUTTON_DATE ).setVisible(True)        
Beispiel #3
0
def getInfo(item, params, src):
    src = enc.smart_unicode(src).encode('utf-8')
    paramArr = __parseParams(params)
    paramPage = paramArr[0].replace('%s', src)

    paramPage = urllib.unquote(paramPage)
    if paramPage.startswith('@') and paramPage.endswith('@'):
        paramPage = item.getInfo(paramPage.strip('@'))

    
    paramRegex = paramArr[1].replace('%s', src)
    if paramRegex.startswith('@') and paramRegex.endswith('@'):
        paramRegex = item.getInfo(paramRegex.strip('@'))
        
    referer = ''
    variables=[]
    if len(paramArr) > 2:
        referer = paramArr[2]
        referer = referer.replace('%s', src)
        if referer.startswith('@') and referer.endswith('@'):
            referer = item.getInfo(referer.strip('@'))
    if len(paramArr) > 3:
        variables = paramArr[3].strip("'").split('|')
    common.log('Get Info from: "'+ paramPage + '" from "' + referer + '"')
    data = common.getHTML(paramPage, referer, referer!='')
    return reg.parseText(data, paramRegex, variables)
def discover_new_runs(denied_runs, conf):
    """Discover new runs.

    Arguments:
        conf: configuration object
    """

    #
    # Discover new run
    #

    run_already_discovered = load_processed_run_ids(conf)

    if common.is_conf_value_equals_true(FIRST_BASE_REPORT_STEP_KEY, conf):
        for run_id in (get_available_new_run_ids(conf) - run_already_discovered - denied_runs):
            aozan.welcome(conf)
            common.log('INFO',
                       'First base report ' + run_id + ' on sequencer ' + common.get_instrument_name(run_id, conf),
                       conf)
            if send_report(run_id, conf):
                add_run_id_to_processed_run_ids(run_id, conf)
                run_already_discovered.add(run_id)

            # Verify space needed during the first base report
            estimate_space_needed.estimate(run_id, conf)
Beispiel #5
0
def test_user_role_list():
    log('user_manage', '角色管理测试开始')
    data = {'role': 4}
    response = global_params.post('ucenter/role/list', data)
    assert response['status'] == 0
    log('user_manage', '角色管理测试结束')
    pass
Beispiel #6
0
def replaceFromDict(dictFilePath, wrd):

    dictionary = enc.smart_unicode(getFileContent(dictFilePath))
    dictionary = dictionary.replace('\r\n','\n')

    p_reg = re.compile('^[^\r\n]+$', re.IGNORECASE + re.DOTALL + re.MULTILINE)
    m_reg = p_reg.findall(dictionary)

    word = enc.smart_unicode(wrd).replace(u'Ãœ','Ü').replace(u'Ö','Ö').replace(u'Ä','Ä')
    try:
        if m_reg and len(m_reg) > 0:
            index = ''
            words = []
            newword = ''
            for m in m_reg:
                if not m.startswith(' '):
                    index = m
                    del words[:]
                else:
                    replWord = m.strip()
                    words.append(replWord)
                    if word.find(' ') != -1:
                        newword = word.replace(replWord,index)

                if (word in words) or (word == index):
                    return index

            if newword != '' and newword != word:
                return newword
    except:
        common.log('Skipped Replacement: ' + word)

    return word
Beispiel #7
0
def test_calendar_checkin_create():
    log('calendar', "签到开始")
    data = {'address': '上海市杨浦区周家嘴路3388号', 'city_name': '上海市', 'content': '测试', 'lat': '31.27692952473959',
            'lng': '121.5386952039931', 'photo_id': '', 'type': 1}
    rs = global_params.post('/calendar/checkin/create', data)
    assert rs['status'] == 0, rs['message']
    log('calendar', "签到成功")
Beispiel #8
0
 def __init__(self, dbg, base_device):
     self.name = name_of_device(base_device)
     t = table(base_device)
     existing = call(dbg, ["dmsetup", "table", self.name ]).strip()
     if existing <> t:
         log(dbg, "Device mapper device %s has table %s, expected %s" % (self.name, existing, t))
         raise (xapi.InternalError("Device mapper device %s has unexpected table" % self.name))
Beispiel #9
0
 def __init__(self, area, token, renew=False):
     self.id = 'radiko'
     self.area = area
     self.token = token
     log('area:%s, token:%s' % (self.area,self.token))
     # 放送局データと設定データを初期化
     self.getStationFile(renew)
Beispiel #10
0
def test_calendar_checkin_detail():
    log('calendar', "签到详情开始")
    data = {"checkin_id": 1}
    rs = global_params.post('/calendar/checkin/detail', data)
    assert rs['status'] == 0, rs['message']
    assert rs['data']['content'] == '测试', rs['message']
    log('calendar', "签到详情成功")
Beispiel #11
0
    def start(self):
        headers = {'pragma': 'no-cache',
            #'X-Radiko-App': 'pc_1',
            #'X-Radiko-App-Version': '2.0.1',
            'X-Radiko-App': 'pc_ts',
            'X-Radiko-App-Version': '4.0.0',
            'X-Radiko-User': '******',
            'X-Radiko-Device': 'pc'}

        req = urllib2.Request(__auth1_url__, headers=headers, data='\r\n')
        auth1fms = urllib2.urlopen(req).info()

        self._response = {}
        self._response['auth_token'] = auth1fms['X-Radiko-AuthToken']
        self._response['key_offset'] = int(auth1fms['X-Radiko-KeyOffset'])
        self._response['key_length'] = int(auth1fms['X-Radiko-KeyLength'])

        '''if __debug__:
            print ("authtoken: %s\noffset: %d length: %d\n" % ( \
                self._response['auth_token'],
                self._response['key_offset'],
                self._response['key_length']))'''
        log("authtoken: %s, offset: %d, length: %d" % ( \
            self._response['auth_token'],
            self._response['key_offset'],
            self._response['key_length']))
Beispiel #12
0
    def start(self, _response):
        self._response = _response

        if 'partial_key' not in self._response or self._response['partial_key'] == '':
            self._response['partial_key'] = self.createPartialKey()

        headers = {'pragma': 'no-cache',
            #'X-Radiko-App': 'pc_1',
            #'X-Radiko-App-Version': '2.0.1',
            'X-Radiko-App': 'pc_ts',
            'X-Radiko-App-Version': '4.0.0',
            'X-Radiko-User': '******',
            'X-Radiko-Device': 'pc',
            'X-Radiko-Authtoken': self._response['auth_token'],
            'X-Radiko-Partialkey': self._response['partial_key']}

        req = urllib2.Request(__auth2_url__, headers=headers, data='\r\n')
        auth2fms = urllib2.urlopen(req).read().decode('utf-8')

        self._response['area_id'] = auth2fms.split(',')[0].strip()
        '''if __debug__:
            print ("authtoken: %s\noffset: %d length: %d \npartialkey: %s\n" % ( \
                self._response['auth_token'],
                self._response['key_offset'],
                self._response['key_length'],
                self._response['partial_key']))'''
        log("authtoken: %s, offset: %d, length: %d, partialkey: %s" % ( \
            self._response['auth_token'],
            self._response['key_offset'],
            self._response['key_length'],
            self._response['partial_key']))
    def _walk(self, path, recursive, types):
        filenames = []
        dirnames = []

        path = xbmc.translatePath(path)
        common.log("Scanner._walk", '"%s"' % path)
        # if xbmcvfs.exists(xbmc.translatePath(path)) or re.match(r"[a-zA-Z]:\\", path) is not None:
        subdirs, files = self.listdir(path)
        for subdir in subdirs:
            dirnames.append(os.path.join(path, subdir))

        for filename in files:
            if types is not None:
                if os.path.splitext(filename)[1].upper() in types or os.path.splitext(filename)[1].lower() in types:
                    filenames.append(os.path.join(path, filename))
                else:
                    common.log("Scanner:_walk", 'Found file "%s" is excluded' % os.path.join(path, filename))
            else:
                filenames.append(os.path.join(path, filename))

        if recursive:
            for item in subdirs:
                dirnames1, filenames1 = self._walk(os.path.join(path, item), recursive, types)
                for item in dirnames1:
                    dirnames.append(item)
                for item in filenames1:
                    filenames.append(item)

        return dirnames, filenames
    def walk(self, path, recursive=False, types=None):
        filenames = []
        dirnames = []
        files_to_return = []
        dirs_to_return = []

        if type(path).__name__ == "unicode":
            path = path.encode("utf-8")

        if path.startswith("multipath://"):
            common.log("Scanner.walk", 'multipath "%s"' % path)
            dirs = path[12:-1].split("/")
            for item in dirs:
                dirnames1, filenames1 = self._walk(urllib.unquote_plus(item), recursive, types)

                for dirname in dirnames1:
                    dirnames.append(dirname)
                for filename in filenames1:
                    filenames.append(filename)

        else:
            common.log("Scanner.walk", 'path "%s"' % path)
            dirnames, filenames = self._walk(path, recursive, types)

        # Make sure everything is a unicode
        for filename in filenames:
            files_to_return.append(common.smart_unicode(filename))
        for dirname in dirnames:
            dirs_to_return.append(common.smart_unicode(dirname))

        return dirs_to_return, files_to_return
    def listdir(self, path):

        try:
            return xbmcvfs.listdir(path)
        except:
            file_list = []
            dir_list = []
            json_response = xbmc.executeJSONRPC(
                '{ "jsonrpc" : "2.0" , "method" : "Files.GetDirectory" , "params" : { "directory" : "%s" , "sort" : { "method" : "file" } } , "id" : 1 }'
                % common.smart_utf8(path.replace("\\", "\\\\"))
            )
            jsonobject = json.loads(json_response)

            try:
                if jsonobject["result"]["files"]:

                    for item in jsonobject["result"]["files"]:

                        filename = common.smart_utf8(item["label"])
                        if item["filetype"] == "directory":
                            dir_list.append(filename)
                        else:
                            file_list.append(filename)

            except Exception, msg:
                common.log("Scanner.listdir", 'Path "%s"' % path, xbmc.LOGERROR)
                common.log("Scanner.listdir", "%s - %s" % (Exception, msg), xbmc.LOGERROR)

            return dir_list, file_list
Beispiel #16
0
def test_calendar_schedule_list():
    log('calendar', "我的日程开始")
    data = {}
    rs = global_params.post('/calendar/schedule/list', data)
    assert rs['status'] == 0, rs['message']
    assert len(rs['data']['list'][0]['calendar_list']) == 2, rs['message']
    log('calendar', "我的日报成功")
Beispiel #17
0
    def parseSwfHead(self):
        global CMN
        self.magic   = self.swfRead(3)
        self.version = ord(self.swfRead(1))
        self.file_length = self.le4Byte(self.swfRead(4))

        rectbits = ord(self.swfRead(1)) >> 3
        total_bytes = int(ceil((5 + rectbits * 4) / 8.0))
        twips_waste = self.swfRead(total_bytes - 1)
        self.frame_rate_decimal = ord(self.swfRead(1))
        self.frame_rate_integer = ord(self.swfRead(1))
        self.frame_count = self.le2Byte(self.swfRead(2))

        '''if __debug__:
            print ("magic: %s\nver: %d\nlen: %d\nframe_rate: %d.%d\ncount: %d\n" % ( \
                self.magic,
                self.version,
                self.file_length,
                self.frame_rate_integer,
                self.frame_rate_decimal,
                self.frame_count))'''
        log("magic: %s, ver: %d, len: %d, frame_rate: %d.%d, count: %d" % ( \
            self.magic,
            self.version,
            self.file_length,
            self.frame_rate_integer,
            self.frame_rate_decimal,
            self.frame_count))
Beispiel #18
0
def test_calendar_checkin_list():
    log('calendar', "我的签到列表开始")
    data = {}
    rs = global_params.post('/calendar/checkin/list', data)
    assert rs['status'] == 0, rs['message']
    assert len(rs['data']['list']) == 1, rs['message']
    log('calendar', "我的签到列表成功")
Beispiel #19
0
    def wait_for_discovered_blades(self):
        log('Wait for discovered blades')
        discovered_macs = []
        restart_times = BLADE_RESTART_TIMES

        for blade in self.node_ids:
            self.blade_node_dict[blade] = None

        with self.ssh:
            all_discovered = self.discovery_waiting_loop(discovered_macs)

        while not all_discovered and restart_times != 0:
            restart_times -= 1
            for blade in self.get_not_discovered_blades():
                self.dha.node_reset(blade)
            with self.ssh:
                all_discovered = self.discovery_waiting_loop(discovered_macs)

        if not all_discovered:
            err('Not all blades have been discovered: %s'
                % self.not_discovered_blades_summary())

        with io.open(self.updated_dea_file) as stream:
            updated_dea = yaml.load(stream)
        updated_dea.update({'blade_node_map': self.blade_node_dict})
        with io.open(self.updated_dea_file, 'w') as stream:
            yaml.dump(updated_dea, stream, default_flow_style=False)
Beispiel #20
0
def test_calendar_daily_detail():
    log('calendar', "日报详情开始")
    data = {"daily_id": 1}
    rs = global_params.post('/calendar/daily/detail', data)
    assert rs['status'] == 0, rs['message']
    assert rs['data']['content'] == '我的日报', rs['message']
    log('calendar', "日报详情成功")
Beispiel #21
0
    def downloadMovie(self, url, path, title, extension):
        if not os.path.exists(path):
            common.log('Path does not exist')
            return None
        if title == '':
            common.log('No title given')
            return None

        file_path = xbmc.makeLegalFilename(os.path.join(path, title + extension))
        file_path = urllib.unquote_plus(file_path)
        # Overwrite existing file?
        if os.path.isfile(file_path):
            self.pDialog = xbmcgui.Dialog()

            if not common.ask('File already exists. Overwrite?\n' + os.path.basename(file_path)):
                title = common.showOSK(urllib.unquote_plus(title), common.translate(30102))
                if not title:
                    return None
                file_path = xbmc.makeLegalFilename(os.path.join(path, title + extension))
                file_path = urllib.unquote_plus(file_path)
        
        success = self.__download(url, file_path)
        if success:
            return file_path
        else:
            return None
Beispiel #22
0
def test_ucenter_level_list():
    log('ucenter', "职级列表开始")
    data = {}
    rs = global_params.post('/ucenter/level/list', data)
    assert rs['status'] == 0, rs['message']
    assert len(rs['data']['list']) == 3, rs['message']
    log('ucenter', "职级列表成功")
def discover_finished_runs(denied_runs, conf):
    """Discover new ended runs

    Arguments:
        conf: configuration object
    """

    run_ids_done = load_processed_run_ids(conf)

    if common.is_conf_value_equals_true(HISEQ_STEP_KEY, conf):
        for run_id in (get_available_finished_run_ids(conf) - run_ids_done - denied_runs):

            if run_id is None or len(run_id) == 0:
                # No run id found
                return []

            aozan.welcome(conf)
            common.log('INFO', 'Ending run detection ' + str(run_id) + ' on ' + common.get_instrument_name(run_id, conf),
                       conf)

            if hiseq_run.get_read_count(run_id, conf) == 0:
                send_failed_run_message(run_id, common.MAX_DELAY_TO_SEND_TERMINATED_RUN_EMAIL, conf)
                add_run_id_to_denied_run_ids(run_id, conf)
                create_run_summary_reports(run_id, conf)
            else:
                if create_run_summary_reports(run_id, conf):
                    send_mail_if_recent_run(run_id, common.MAX_DELAY_TO_SEND_TERMINATED_RUN_EMAIL, conf)
                    add_run_id_to_processed_run_ids(run_id, conf)
                    run_ids_done.add(run_id)

    return run_ids_done
Beispiel #24
0
def test_apply_type_list():
    log('apply_type', "申请单类型列表开始")
    data = {"item_type": "19"}
    rs = global_params.post('/apply/type/list', data)
    assert rs['status'] == 0, rs['message']
    assert len(rs['data']['list']) == 3, rs['message']
    log('apply_type', "申请单类型列表成功")
Beispiel #25
0
def lock_step(lock_file_path, step_name, conf):
    """Lock a step.

    Arguments:
        lock_file_path: lock_file path
        step_name: step name
        conf: configuration object
    """

    # Check if parent directory of the lock file exists
    if not os.path.isdir(os.path.dirname(lock_file_path)):
        common.log('SEVERE', 'Parent directory of lock file does not exist. The lock file for ' + step_name +
                   ' step has not been created: ' + lock_file_path, conf)
        return True

    # Return False if the run is currently processed
    if os.path.isfile(lock_file_path):
        return False

    # Create the lock file
    try:
        open(lock_file_path, 'w').close()
    except:
        common.log('SEVERE', 'The lock file cannot be created (' + sys.exc_info()[0] + ') for ' + step_name +
                   ': ' + lock_file_path, conf)
        return True

    return True
Beispiel #26
0
def parse_arguments():
    parser = ArgParser(prog='python %s' % __file__)
    parser.add_argument('-dha', dest='dha_uri', action='store',
                        default=False, help='dha configuration file FQDN URI', required=True)
    parser.add_argument('-deab', dest='dea_base_uri', action='store',
                        default=False, help='dea base configuration FQDN URI', required=True)
    parser.add_argument('-deao', dest='dea_pod_override_uri', action='store',
                        default=False, help='dea POD override configuration FQDN URI',
                        required=True)
    parser.add_argument('-scenario-base-uri', dest='scenario_base_uri', action='store',
                        default=False, help='Deploymen scenario base directory URI',
                        required=True)
    parser.add_argument('-scenario', dest='scenario', action='store',
                        default=False, help='Deploymen scenario short-name (priority), or base file name (in the absense of a shortname defenition)',
                        required=True)

    parser.add_argument('-plugins', dest='plugins_uri', action='store',
                        default=False, help='Plugin configurations directory URI',
                        required=True)
    parser.add_argument('-output', dest='output_path', action='store',
                        default=False,
                        help='Local path for resulting output configuration files',
                        required=True)
    args = parser.parse_args()
    log(args)
    kwargs = {'dha_uri': args.dha_uri,
              'dea_base_uri': args.dea_base_uri,
              'dea_pod_override_uri': args.dea_pod_override_uri,
              'scenario_base_uri': args.scenario_base_uri,
              'scenario': args.scenario,
              'plugins_uri': args.plugins_uri,
              'output_path': args.output_path}
    return kwargs
Beispiel #27
0
def exception_error(step, desc, conf):
    """Handle an exception.

    Arguments:
        step: error step
        desc: description of the section error
        conf: configuration object
        run_id: run_id
    """
    exp_info = sys.exc_info()
    exception_msg = str(exp_info[0]) + ' (' + str(exp_info[1]) + ')'
    traceback_msg = traceback.format_exc(exp_info[2])

    short_message = desc + ', catch exception ' + exception_msg
    message = desc + ', catch exception ' + exception_msg + \
              '\n Stacktrace: \n' + traceback_msg

    if step == 'sync':
        sync_run.error(short_message, message, conf)
    elif step == 'demux':
        demux_run.error(short_message, message, conf)
    elif step == 'qc':
        qc_run.error(short_message, message, conf)
    else:
        # Log the exception
        common.log('SEVERE', 'Exception: ' + exception_msg, conf)
        common.log('WARNING', traceback_msg.replace('\n', ' '), conf)

        # Send a mail with the exception
        common.send_msg("[Aozan] Exception: " + exception_msg, traceback_msg,
                        True, conf)
Beispiel #28
0
    def getSteamPath(self):
        import errno, sys, _winreg
        steamPath = ''

        try:
            if sys.maxsize > 2**32:
                arch_keys = {_winreg.KEY_WOW64_32KEY, _winreg.KEY_WOW64_64KEY}
            else:
                arch_keys = {_winreg.KEY_READ}

            for arch_key in arch_keys:

                key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Valve\Steam", 0, _winreg.KEY_READ | arch_key)
                try:
                    steamPath = _winreg.QueryValueEx(key, 'InstallPath')[0]
                except OSError as e:
                    if e.errno == errno.ENOENT:
                        # DisplayName doesn't exist in this skey
                        pass
                finally:
                    key.Close()
        except Exception,e:
            log(e.message)
            log(e.__class__.__name__)
            traceback.print_exc(e)
Beispiel #29
0
def test_ucenter_user_invite():
    log('cost_center', "邀请员工开始")
    data = {"fullname": "帅哥", "email": "*****@*****.**", "superior": "-1", "cost_center_id": 13, "level_id": 2,
            "telephone": "18616369919"}
    rs = global_params.post('/ucenter/user/invite', data)
    assert rs['status'] == 0, rs['message']
    log('cost_center', "邀请员工成功")
    def parse(self, lItem):
        url = lItem['url']
        cfg = lItem['cfg']
        ext = getFileExtension(url)

        successfullyScraped = True

        tmpList = None
        if lItem['catcher']:
            catcher = lItem['catcher']
            cfg = os.path.join(common.Paths.catchersDir, '__' + catcher + '.cfg')
            tmpList = self.__loadLocal(cfg, lItem)
            if tmpList and len(tmpList.rules) > 0:
                successfullyScraped = self.__loadRemote(tmpList, lItem)
        else:
            if ext == 'cfg':
                tmpList = self.__loadLocal(url, lItem)
                if tmpList and tmpList.start != '' and len(tmpList.rules) > 0:
                    lItem['url'] = tmpList.start
                    successfullyScraped = self.__loadRemote(tmpList, lItem)
            elif cfg:
                tmpList = self.__loadLocal(cfg, lItem)
                if tmpList and len(tmpList.rules) > 0:
                    successfullyScraped = self.__loadRemote(tmpList, lItem)

        # autoselect
        if tmpList and tmpList.skill.find('autoselect') != -1 and len(tmpList.items) == 1:
            m = tmpList.items[0]
            m_type = m['type']

            if m_type == 'rss':
                common.log('Autoselect - ' + m['title'])
                lItem = m
                tmpList = self.parse(lItem).list

        if not tmpList:
            return ParsingResult(ParsingResult.Code.CFGSYNTAX_INVALID, None)
        if tmpList and successfullyScraped == False:
            return ParsingResult(ParsingResult.Code.WEBREQUEST_FAILED, tmpList)

        # Remove duplicates

        if tmpList.skill.find('allowDuplicates') == -1:
            urls = []
            for i in range(len(tmpList.items)-1,-1,-1):
                item = tmpList.items[i]

                tmpUrl = item['url']
                tmpCfg = item['cfg']
                if not tmpCfg:
                    tmpCfg = ''

                if not urls.__contains__(tmpUrl + '|' + tmpCfg):
                    urls.append(tmpUrl + '|' + tmpCfg)
                else:
                    if item['type'] !='say':
                        tmpList.items.remove(item)


        return ParsingResult(ParsingResult.Code.SUCCESS, tmpList)
Beispiel #31
0
 async def process_stop_and_limit_price_pairs(self, symbol):
     assert symbol not in self.pending_order_request or self.pending_order_request[
         symbol] == 0
     reduction_amount = self.get_total_order_amount(
         query_orders(symbol, -self.positions[symbol]['amount'], 'STOP',
                      999))
     expansion_amount = self.get_total_order_amount(
         query_orders(symbol, self.positions[symbol]['amount'], 'LIMIT',
                      999))
     log(reduction_amount, expansion_amount)
     if abs(reduction_amount) > abs(expansion_amount):
         log('REDUCTION EXCEEDS EXPANSION!')
         self.stop_and_limit_price_pairs[symbol] = []
     else:
         if len(self.stop_and_limit_price_pairs[symbol]) > 0:
             order_info = self.stop_and_limit_price_pairs[symbol].pop(0)
             await self.place_stop_order(symbol, -order_info[0],
                                         order_info[1], False,
                                         order_info[3])
             await self.place_limit_order(symbol, order_info[0],
                                          order_info[2], False,
                                          order_info[3])
             log(self.stop_and_limit_price_pairs)
Beispiel #32
0
 def __init__(self):
  myplatform=platform()
  log("XvBMC_Platform: "+str(myplatform))
  if not myplatform=='linux':
   dialog.ok(MainTitle+SubTitle,subtitleNope,nonlinux,nonelecNL)
   log("none Linux OS ie. Open-/LibreELEC")
  else:
   log("linux os")
   if dialog.yesno("XvBMC-NL LibreELEC \'[COLOR white]OS[/COLOR]-update\'",'Preparing [COLOR red]OE[/COLOR]-[B]2[/B]-[COLOR lime]LE[/COLOR] v'+LibreVersie+' and reboot when done...'):
    url='http://releases.libreelec.tv/LibreELEC-RPi2.arm-'+LibreVersie+'.tar'
    path=xbmc.translatePath(os.path.join('/storage/.update/',''))
    dp=xbmcgui.DialogProgress()
    dp.create("XvBMC Nederland","XvBMC-DEV: doing some VoOdOo...",'','Please Wait')
    lib=os.path.join(path,'elec'+LibreVersie+'.tar')
    try:
     os.remove(lib)
    except:
     pass
    downloader.download(url,lib)
    time.sleep(3)
    dialog.ok(MainTitle,'[COLOR red]Open[/COLOR]-[B]2[/B]-[COLOR lime]Libre[/COLOR]ELEC System-Fork finished!','','Press OK to reboot...')
    xbmc.sleep(1000)
    xbmc.executebuiltin("Reboot")
Beispiel #33
0
    async def cancel_orders(self, group_id, symbol, long, short):
        if self.orders is None or symbol not in self.orders:
            return

    #     log('orders({}): {}'.format(len(self.orders), self.orders))
        i = 0
        for order_id in self.orders[symbol]:
            order = self.orders[symbol][order_id]
            log('order:', order)
            if group_id is not None:
                if group_id != 0 and order[
                        'gid'] % 1000 != group_id if group_id == 0 else order[
                            'gid'] is not None:
                    continue
            log(i)
            if (long and order['amount'] > 0) or (short
                                                  and order['amount'] < 0):
                i += 1
                r = post('/v1/order/cancel', {'order_id': order_id})
                log(r)
                if symbol in self.pending_order_request:
                    self.pending_order_request[symbol] += 1
                else:
                    self.pending_order_request[symbol] = 1
Beispiel #34
0
    def __loadRemote(self, inputList, lItem):

        try:
            inputList.curr_url = lItem['url']

            count = 0
            i = 1
            maxits = 2  # 1 optimistic + 1 demystified
            ignoreCache = False
            demystify = False
            startUrl = inputList.curr_url
            #print inputList, lItem
            while count == 0 and i <= maxits:
                if i > 1:
                    ignoreCache = True
                    demystify = True

                # Trivial: url is from known streamer
                items = self.__parseHtml(inputList.curr_url,
                                         '"' + inputList.curr_url + '"',
                                         inputList.rules, inputList.skill,
                                         inputList.cfg, lItem)
                count = len(items)

                # try to find items in html source code
                if count == 0:
                    referer = ''
                    if lItem['referer']:
                        referer = lItem['referer']
                    data = common.getHTML(inputList.curr_url, referer,
                                          ignoreCache, demystify)
                    if data == '':
                        return False

                    msg = 'Remote URL ' + str(inputList.curr_url) + ' opened'
                    if demystify:
                        msg += ' (demystified)'
                    common.log(msg)

                    if inputList.section != '':
                        section = inputList.section
                        data = self.__getSection(data, section)

                    if lItem['section']:
                        section = lItem['section']
                        data = self.__getSection(data, section)

                    items = self.__parseHtml(inputList.curr_url, data,
                                             inputList.rules, inputList.skill,
                                             inputList.cfg, lItem)
                    count = len(items)
                    common.log('    -> ' + str(count) + ' item(s) found')

                # find rtmp stream
                #common.log('Find rtmp stream')
                if count == 0:
                    item = self.__findRTMP(data, startUrl, lItem)
                    if item:
                        items = []
                        items.append(item)
                        count = 1

                # find embedding javascripts
                #common.log('Find embedding javascripts')
                if count == 0:
                    item = findJS(data)
                    if item:
                        firstJS = item[0]
                        streamId = firstJS[0]
                        jsUrl = firstJS[1]
                        streamerName = getHostName(jsUrl)
                        jsSource = getHTML(jsUrl, startUrl, True, False)
                        phpUrl = findPHP(jsSource, streamId)
                        if phpUrl:
                            data = getHTML(phpUrl, startUrl, True, True)
                            item = self.__findRTMP(data, phpUrl, lItem)
                            if item:

                                if streamerName:
                                    item['title'] = item['title'].replace(
                                        'RTMP', streamerName)

                                items = []
                                items.append(item)
                                count = 1

                # find vcods
                #common.log('find vcods')
                if count == 0:
                    vcods = findVCods(data)
                    if vcods:
                        sUrl = vcods[0]
                        cod1 = vcods[1]
                        cod2 = vcods[2]
                        swfUrl = vcods[3]
                        unixTS = str(dt.getUnixTimestamp())
                        sUrl = sUrl + '?callback=jQuery1707757964063647694_1347894980192&v_cod1=' + cod1 + '&v_cod2=' + cod2 + '&_=' + unixTS
                        tmpData = getHTML(sUrl, urllib.unquote_plus(startUrl),
                                          True, False)
                        if tmpData and tmpData.find("Bad Request") == -1:
                            newReg = '"result1":"([^\"]+)","result2":"([^\"]+)"'
                            link = regexUtils.findall(tmpData, newReg)
                            if link:
                                _file = link[0][0]
                                rtmp = link[0][1].replace('\\', '')
                                #.replace('/redirect','/vod')
                                item = CListItem()
                                item['title'] = getHostName(
                                    sUrl) + '* - ' + _file
                                item['type'] = 'video'
                                item[
                                    'url'] = rtmp + ' playPath=' + _file + ' swfUrl=' + swfUrl + ' swfVfy=1 live=true pageUrl=' + startUrl
                                item.merge(lItem)
                                items.append(item)
                                count = 1

                # find redirects
                #common.log('find redirects')
                if count == 0:
                    red = self.__findRedirect(startUrl, inputList.curr_url)
                    if startUrl == red:
                        common.log('    -> No redirect found')
                    else:
                        common.log('    -> Redirect: ' + red)
                        inputList.curr_url = red
                        common.log(
                            str(len(inputList.items)) + ' items ' +
                            inputList.cfg + ' -> ' + red)
                        startUrl = red
                        if lItem['referer']:
                            lItem['referer'] = red
                        i = 0

                i += 1

            if count != 0:
                inputList.items = inputList.items + items

        except IOError:
            if common.enable_debug:
                traceback.print_exc(file=sys.stdout)
            return False
        return True
Beispiel #35
0
    def __parseHtml(self, url, data, rules, skills, definedIn, lItem):          

        common.log('_parseHtml called' + url)
        items = []

        for item_rule in rules:
            common.log('rule: ' + item_rule.infos)
            #print('>>>>item_rule: ',item_rule)
            #print("MMMMMMMM",mydump(item_rule))

            # Zmiana MN
            if not hasattr(item_rule, 'precheck') or (item_rule.precheck in data):

                revid = re.compile(item_rule.infos, re.IGNORECASE + re.DOTALL + re.MULTILINE + re.UNICODE)
                for reinfos in revid.findall(data):
                    tmp = CListItem()
                  
                    if lItem['referer']:
                        tmp['referer'] = lItem['referer']
                      
                    if item_rule.order.find('|') != -1:
                        infos_names = item_rule.order.split('|')
                        infos_values = list(reinfos)
                        i = 0
                        for name in infos_names:
                            tmp[name] = infos_values[i]
                            i = i+1
                    else:
                        tmp[item_rule.order] = reinfos

                    for info in item_rule.info_list:
                        info_value = tmp[info.name]
                        if info_value:
                            if info.build.find('%s') != -1:
                                tmpVal = info.build % info_value
                                tmp[info.name] = tmpVal
                            continue

                        if info.build.find('%s') != -1:
                            if info.src.__contains__('+'):
                                tmpArr = info.src.split('+')
                                src = ''
                                for t in tmpArr:
                                    t = t.strip()
                                    if t.find('\'') != -1:
                                        src = src + t.strip('\'')
                                    else:
                                        src = src + tmp[t]
                            elif info.src.__contains__('||'):
                                variables = info.src.split('||')
                                src = firstNonEmpty(tmp, variables)
                            else:
                                src = tmp[info.src]

                            if src and info.convert != []: 
                                tmp['referer'] = url                              
                                src = self.__parseCommands(tmp, src, info.convert)
                                if isinstance(src, dict):
                                    for dKey in src:
                                        tmp[dKey] = src[dKey]
                                    src = src.values()[0]

                            info_value = info.build % (src)
                        else:
                            info_value = info.build

                        tmp[info.name] = info_value

                    if tmp['url']:
                        tmp['url'] = item_rule.url_build % (tmp['url'])
                    else:
                        tmp['url'] = url
                    
                    tmp.merge(lItem)
                    if item_rule.skill.find('append') != -1:
                        tmp['url'] = url + tmp['url']

                    if item_rule.skill.find('space') != -1:
                        tmp['title'] = ' %s ' % tmp['title'].strip()

                    if skills.find('videoTitle') > -1:
                        tmp['videoTitle'] = tmp['title']

                    tmp['definedIn'] = definedIn
                    items.append(tmp)

        return items
Beispiel #36
0
    def __loadRemote(self, inputList, lItem):

        try:
            inputList.curr_url = lItem['url']

            count = 0
            i = 1
            maxits = 2      # 1 optimistic + 1 demystified
            ignoreCache = False
            demystify = False
            back = ''
            startUrl = inputList.curr_url
            #print inputList, lItem
            while count == 0 and i <= maxits:
                if i > 1:
                    ignoreCache = True
                    demystify =  True

                # Trivial: url is from known streamer
                if back:
                    lItem['referer'] = back
                items = self.__parseHtml(inputList.curr_url, '"' + inputList.curr_url + '"', inputList.rules, inputList.skill, inputList.cfg, lItem)
                count = len(items)


                # try to find items in html source code
                if count == 0:
                    referer = ''
                    if lItem['referer']:
                        referer = lItem['referer']
                    data = common.getHTML(inputList.curr_url, None, referer, False, False, ignoreCache, demystify)
                    if data == '':
                        return False

                    msg = 'Remote URL ' + inputList.curr_url + ' opened'
                    if demystify:
                        msg += ' (demystified)'
                    common.log(msg)

                    
                    if inputList.section != '':
                        section = inputList.section
                        data = self.__getSection(data, section)
                        
                    if lItem['section']:
                        section = lItem['section']
                        data = self.__getSection(data, section)
                                                
                    
                    items = self.__parseHtml(inputList.curr_url, data, inputList.rules, inputList.skill, inputList.cfg, lItem)
                    count = len(items)
                    common.log('    -> ' + str(count) + ' item(s) found')

                # find rtmp stream
                common.log('Find rtmp stream')
                if count == 0:
                    item = self.__findRTMP(data, startUrl, lItem)
                    if item:
                        items = []
                        items.append(item)
                        count = 1

                # find embedding javascripts
                common.log('Find embedding javascripts')
                if count == 0:
                    item = findJS(data)
                    if item:
                        firstJS = item[0]
                        streamId = firstJS[0]
                        jsUrl = firstJS[1]
                        if not jsUrl.startswith('http://'):
                            jsUrl = urllib.basejoin(startUrl,jsUrl)
                        streamerName = getHostName(jsUrl)
                        jsSource = getHTML(jsUrl, None, startUrl)
                        phpUrl = findPHP(jsSource, streamId)
                        if phpUrl:
                            data = getHTML(phpUrl, None, startUrl)
                            item = self.__findRTMP(data, phpUrl, lItem)
                            if item:
                                
                                if streamerName:
                                    item['title'] = item['title'].replace('RTMP', streamerName)
                                
                                items = []
                                items.append(item)
                                count = 1
                            else:
                                red = phpUrl
                                common.log('    -> Redirect: ' + red)
                                if back == red:
                                    break
                                back = inputList.curr_url
                                inputList.curr_url = red
                                common.log(str(len(inputList.items)) + ' items ' + inputList.cfg + ' -> ' + red)
                                startUrl = red
                                continue


                # find redirects
                common.log('find redirects')
                if count == 0:
                    red = self.__findRedirect(startUrl, inputList.curr_url)
                    if startUrl == red:
                        common.log('    -> No redirect found')
                    else:
                        common.log('    -> Redirect: ' + red)
                        if back == red:
                            break
                        back = inputList.curr_url
                        inputList.curr_url = red
                        common.log(str(len(inputList.items)) + ' items ' + inputList.cfg + ' -> ' + red)
                        startUrl = red
                        i = 0

                i += 1


            if count != 0:
                inputList.items = inputList.items + items


        except:
            traceback.print_exc(file = sys.stdout)
            return False
        return True
Beispiel #37
0
    def parseView(self, lItem):
        def endOfDirectory(succeeded=True):
            if self.handle > -1:
                xbmcplugin.endOfDirectory(handle=self.handle,
                                          succeeded=succeeded,
                                          cacheToDisc=True)
            else:
                common.log('Handle -1')

        if not lItem:
            endOfDirectory(False)
            return None

        if lItem['type'] == 'search':
            search_phrase = self.getSearchPhrase()
            if not search_phrase:
                common.log("search canceled")
                endOfDirectory(False)
                return None
            else:
                lItem['type'] = 'rss'
                lItem['url'] = lItem['url'] % (
                    urllib.quote_plus(search_phrase))

        url = lItem['url']

        if url == common.Paths.customModulesFile:
            self.customModulesManager.getCustomModules()

        tmpList = None
        result = self.parser.parse(lItem)
        if result.code == ParsingResult.Code.SUCCESS:
            tmpList = result.list
        elif result.code == ParsingResult.Code.CFGFILE_NOT_FOUND:
            common.showError("No encuentro el fichero Cfg")
            endOfDirectory(False)
            return None
        elif result.code == ParsingResult.Code.CFGSYNTAX_INVALID:
            common.showError("sintaxis invalida en Cfg")
            endOfDirectory(False)
            return None
        elif result.code == ParsingResult.Code.WEBREQUEST_FAILED:
            common.showError("Ha fallado la llamada a la Web")
            if len(result.list.items) > 0:
                tmpList = result.list
            else:
                endOfDirectory(False)
                return None

        # if it's the main menu, add folder 'Favourites' and 'Custom Modules
        if url == self.MAIN_MENU_FILE:
            tmp = ListItem.create()
            tmp['title'] = ' [COLOR blue]ViendoKodi[/COLOR] [COLOR red]Streaming[/COLOR]'
            tmp['type'] = 'say'
            tmp['url'] = ''
            tmp['icon'] = os.path.join(common.Paths.imgDir, 'icon.png')
            tmpList.items.insert(0, tmp)
            tmp = ListItem.create()
            tmp['title'] = '[COLOR red][B] [/B][/COLOR]'
            tmp['type'] = 'say'
            tmp['url'] = ''
            tmp['icon'] = os.path.join(common.Paths.imgDir, 'icon.png')

            tmpList.items.insert(1, tmp)

            #tmp = ListItem.create()
            #tmp['title'] = '[COLOR red]Custom Modules[/COLOR]'
            #tmp['type'] = 'rss'
            #tmp['url'] = os.path.join(common.Paths.customModulesDir, 'custom.cfg')
            #tmpList.items.insert(0, tmp)

        # if it's the favourites menu, add item 'Add item'
        elif url == common.Paths.favouritesFile or url.startswith(
                'favfolders'):

            if url.startswith('favfolders'):
                url = os.path.normpath(
                    os.path.join(common.Paths.favouritesFolder, url))

            tmp = ListItem.create()
            tmp['title'] = 'Add item...'
            tmp['type'] = 'command'
            tmp['icon'] = os.path.join(common.Paths.imgDir, 'bookmark_add.png')
            action = 'RunPlugin(%s)' % (self.base + '?mode=' +
                                        str(Mode.ADDITEM) + '&url=' + url)
            tmp['url'] = action
            tmpList.items.append(tmp)

        # Create menu or play, if it's a single video and autoplay is enabled
        count = len(tmpList.items)
        if (count == 0 and not url.startswith('favfolders')):
            common.showInfo('No stream available')
            #Directory with 0 items
            endOfDirectory(False)
        elif not (common.getSetting('autoplay') == 'true' and count == 1
                  and len(tmpList.getVideos()) == 1):
            # sort methods
            sortKeys = tmpList.sort.split('|')
            setSortMethodsForCurrentXBMCList(self.handle, sortKeys)

            # Add items to XBMC list
            for m in tmpList.items:
                self.addListItem(m, len(tmpList.items))
            #Directory with >1 items
            endOfDirectory(True)
        else:
            #Directory with 0 items
            endOfDirectory(False)
        return tmpList
Beispiel #38
0
import xmltodict
import sys
import os
import shutil
import traceback

import logging
from queue import Queue
from threading import Thread, Lock
from common import pretty_print, log

try:
    import yara
    ENABLE_YARA = True
except (OSError, ImportError):
    log("Warning: Yara is not properly installed, will not enable",
        logging.WARNING)
    ENABLE_YARA = False

bucket_q = Queue()
download_q = Queue()

grep_list = None

arguments = None

yara_rules = None

#Disable requests logging
logging.getLogger("requests").setLevel(logging.WARNING)

Beispiel #39
0
    def findGames(self, orgGames, games):
        xbmc.executebuiltin("Dialog.Close(busydialog)")
        isCanceled = False
        try:
            gameData = self.getMfstFiles()
            numFiles = len(gameData)
            if (numFiles > 0):
                counter = 0

                pDialog = xbmcgui.DialogProgress()
                pDialog.create(__addonname__, localize(33003))
                pDialog.update(0, localize(33003))

                for game in gameData:

                    try:
                        installPath = self.getGameInstallPath(game)
                        #log("installPath = " + installPath)
                        fileName = self.extractDLL(installPath)
                        xmlPath = os.path.join(TEMP_DIR, 'Data_1.bin')
                        name, id, path = self.parseXML(xmlPath, installPath,
                                                       fileName)

                        gameId = 'OR-%s' % id

                        orgGame = None

                        for game in orgGames:
                            if (game.gameId == gameId):
                                orgGame = game
                                break

                        isNameChanged = 0
                        isPathChanged = 0
                        isIconChanged = 0
                        isFanartChanged = 0
                        thumbImage = ''
                        fanartImage = ''
                        type = 2

                        #log(path)
                        if ((not orgGame) or (orgGame is not None
                                              and orgGame.isFanartChanged == 0
                                              and orgGame.isIconChanged == 0)):
                            gameFiles = self.downloadGameImage(
                                gameId, name, path)
                            thumbImage = gameFiles[0]
                            fanartImage = gameFiles[1]
                        elif (orgGame is not None
                              and orgGame.isFanartChanged == 0):
                            gameFiles = self.downloadGameImage(
                                gameId, name, path)
                            thumbImage = orgGame.thumbImage
                            fanartImage = gameFiles[1]
                            isIconChanged = 1
                        elif (orgGame is not None
                              and orgGame.isIconChanged == 0):
                            gameFiles = self.downloadGameImage(
                                gameId, name, path)
                            thumbImage = gameFiles[0]
                            fanartImage = orgGame.fanartImage
                            isFanartChanged = 1
                        else:
                            thumbImage = orgGame.thumbImage
                            fanartImage = orgGame.fanartImage
                            isIconChanged = 1
                            isFanartChanged = 1

                        if (orgGame is not None
                                and orgGame.isNameChanged == 1):
                            name = orgGame.title
                            isNameChanged = 1

                        if (orgGame is not None
                                and orgGame.isPathChanged == 1):
                            name = orgGame.path
                            isPathChanged = 1

                        games[gameId] = {}
                        games[gameId]['name'] = name
                        games[gameId]['path'] = path
                        games[gameId]['isNameChanged'] = isNameChanged
                        games[gameId]['isPathChanged'] = isPathChanged
                        games[gameId]['isIconChanged'] = isIconChanged
                        games[gameId]['isFanartChanged'] = isFanartChanged
                        games[gameId]['type'] = type
                        games[gameId]['thumbImage'] = thumbImage
                        games[gameId]['fanartImage'] = fanartImage

                        counter = counter + 1

                        if (pDialog.iscanceled()):
                            isCanceled = True
                            break

                        pDialog.update(
                            int((float(counter) / float(numFiles)) * 100),
                            localize(33003))

                    except:
                        log("error occured")

                pDialog.close()
        except Exception, e:
            #log(e.message)
            #log(e.__class__.__name__)
            traceback.print_exc(e)
def main():
    # Get arguments
    args, kwargs = splunk.Intersplunk.getKeywordsAndOptions()

    # Enable debugging by passing 'debug=yes' as an argument of
    # the command on the Splunk searchbar.

    debug = common.check_debug(kwargs)

    if len(args) < 2:
        logger.error(
            "pancontentpack: Wrong number of arguments: %s, expected 2.\n" %
            len(args))
        usage()

    if args[1] == "apps":
        logger.info(
            "Getting apps from content pack on Palo Alto Networks device at %s..."
            % args[0])
    elif args[1] == "threats":
        logger.info(
            "Getting threats from content pack on Palo Alto Networks device at %s..."
            % args[0])
    else:
        usage()

    # Results contains the data from the search results and settings
    # contains the sessionKey that we can use to talk to Splunk
    # Ignore the results
    results, unused1, settings = splunk.Intersplunk.getOrganizedResults()
    # Get the sessionKey
    sessionKey = settings['sessionKey']

    log(debug, "Begin get API key")
    # Get the API key from the Splunk store or from the device at hostname if no apikey is stored
    apikey = common.apikey(sessionKey, args[0], debug)

    device = pandevice.base.PanDevice(args[0], api_key=apikey)
    device.refresh_system_info()

    try:
        if args[1] == "apps":
            device.xapi.get("/config/predefined/application")
            app_xml = device.xapi.xml_document
            csv = parse_apps(app_xml)
        else:
            if device._version_info >= (8, 0, 0):
                threat_xml = device.op(
                    'show predefined xpath "/predefined/threats"',
                    xml=True,
                    cmd_xml=True,
                )
            else:
                device.xapi.get("/config/predefined/threats")
                threat_xml = device.xapi.xml_document
            csv = parse_threats(threat_xml)

    except pan.xapi.PanXapiError as e:
        common.exit_with_error(string_types(e))

    # output results
    splunk.Intersplunk.outputResults(csv)
Beispiel #41
0
def main():
    warning = "Deleting the backup catalog may have unexpected consequences. Operational issues are unknown."
    common.log("WARNING: %s" % warning, log_type="!")
    time.sleep(5)

    common.execute(["wbadmin", "delete", "catalog", "-quiet"])
    def __getitem__(self, index):
        imgfile, objs = self.items[index]
        image = common.imread(imgfile)

        if image is None:
            log.info("{} is empty, index={}".format(imgfile, index))
            return self[random.randint(0, len(self.items) - 1)]

        keepsize = 12
        image, objs = augment.webface(image,
                                      objs,
                                      self.width,
                                      self.height,
                                      keepsize=0)

        # norm
        image = ((image / 255.0 - self.mean) / self.std).astype(np.float32)

        posweight_radius = 2
        stride = 4
        fm_width = self.width // stride
        fm_height = self.height // stride

        heatmap_gt = np.zeros((1, fm_height, fm_width), np.float32)
        heatmap_posweight = np.zeros((1, fm_height, fm_width), np.float32)
        keep_mask = np.ones((1, fm_height, fm_width), np.float32)
        reg_tlrb = np.zeros((1 * 4, fm_height, fm_width), np.float32)
        reg_mask = np.zeros((1, fm_height, fm_width), np.float32)
        distance_map = np.zeros((1, fm_height, fm_width), np.float32) + 1000
        landmark_gt = np.zeros((1 * 10, fm_height, fm_width), np.float32)
        landmark_mask = np.zeros((1, fm_height, fm_width), np.float32)

        hassmall = False
        for obj in objs:
            isSmallObj = obj.area < keepsize * keepsize

            if isSmallObj:
                cx, cy = obj.safe_scale_center(1 / stride, fm_width, fm_height)
                keep_mask[0, cy, cx] = 0
                w, h = obj.width / stride, obj.height / stride

                x0 = int(common.clip_value(cx - w // 2, fm_width - 1))
                y0 = int(common.clip_value(cy - h // 2, fm_height - 1))
                x1 = int(common.clip_value(cx + w // 2, fm_width - 1) + 1)
                y1 = int(common.clip_value(cy + h // 2, fm_height - 1) + 1)
                if x1 - x0 > 0 and y1 - y0 > 0:
                    keep_mask[0, y0:y1, x0:x1] = 0
                hassmall = True

        for obj in objs:

            classes = 0
            cx, cy = obj.safe_scale_center(1 / stride, fm_width, fm_height)
            reg_box = np.array(obj.box) / stride
            isSmallObj = obj.area < keepsize * keepsize

            if isSmallObj:
                if obj.area >= 5 * 5:
                    distance_map[classes, cy, cx] = 0
                    reg_tlrb[classes * 4:(classes + 1) * 4, cy, cx] = reg_box
                    reg_mask[classes, cy, cx] = 1
                continue

            w, h = obj.width / stride, obj.height / stride
            x0 = int(common.clip_value(cx - w // 2, fm_width - 1))
            y0 = int(common.clip_value(cy - h // 2, fm_height - 1))
            x1 = int(common.clip_value(cx + w // 2, fm_width - 1) + 1)
            y1 = int(common.clip_value(cy + h // 2, fm_height - 1) + 1)
            if x1 - x0 > 0 and y1 - y0 > 0:
                keep_mask[0, y0:y1, x0:x1] = 1

            w_radius, h_radius = common.truncate_radius(
                (obj.width, obj.height))
            gaussian_map = common.draw_truncate_gaussian(
                heatmap_gt[classes, :, :], (cx, cy), h_radius, w_radius)

            mxface = 300
            miface = 25
            mxline = max(obj.width, obj.height)
            gamma = (mxline - miface) / (mxface - miface) * 10
            gamma = min(max(0, gamma), 10) + 1
            common.draw_gaussian(heatmap_posweight[classes, :, :], (cx, cy),
                                 posweight_radius,
                                 k=gamma)

            range_expand_x = math.ceil(w_radius)
            range_expand_y = math.ceil(h_radius)

            min_expand_size = 3
            range_expand_x = max(min_expand_size, range_expand_x)
            range_expand_y = max(min_expand_size, range_expand_y)

            icx, icy = cx, cy
            reg_landmark = None
            fill_threshold = 0.3

            if obj.haslandmark:
                reg_landmark = np.array(obj.x5y5_cat_landmark) / stride
                x5y5 = [cx] * 5 + [cy] * 5
                rvalue = (reg_landmark - x5y5)
                landmark_gt[0:10, cy, cx] = np.array(common.log(rvalue)) / 4
                landmark_mask[0, cy, cx] = 1

            if not obj.rotate:
                for cx in range(icx - range_expand_x,
                                icx + range_expand_x + 1):
                    for cy in range(icy - range_expand_y,
                                    icy + range_expand_y + 1):
                        if cx < fm_width and cy < fm_height and cx >= 0 and cy >= 0:

                            my_gaussian_value = 0.9
                            gy, gx = cy - icy + range_expand_y, cx - icx + range_expand_x
                            if gy >= 0 and gy < gaussian_map.shape[
                                    0] and gx >= 0 and gx < gaussian_map.shape[
                                        1]:
                                my_gaussian_value = gaussian_map[gy, gx]

                            distance = math.sqrt((cx - icx)**2 + (cy - icy)**2)
                            if my_gaussian_value > fill_threshold or distance <= min_expand_size:
                                already_distance = distance_map[classes, cy,
                                                                cx]
                                my_mix_distance = (
                                    1 - my_gaussian_value) * distance

                                if my_mix_distance > already_distance:
                                    continue

                                distance_map[classes, cy, cx] = my_mix_distance
                                reg_tlrb[classes * 4:(classes + 1) * 4, cy,
                                         cx] = reg_box
                                reg_mask[classes, cy, cx] = 1

        # if hassmall:
        #     common.imwrite("test_result/keep_mask.jpg", keep_mask[0]*255)
        #     common.imwrite("test_result/heatmap_gt.jpg", heatmap_gt[0]*255)
        #     common.imwrite("test_result/keep_ori.jpg", (image*self.std+self.mean)*255)
        return T.to_tensor(
            image
        ), heatmap_gt, heatmap_posweight, reg_tlrb, reg_mask, landmark_gt, landmark_mask, len(
            objs), keep_mask
Beispiel #43
0
def Put(i):
    global head_client
    global ack_pubsub
    global fails_since_last_success

    i_str = str(i)  # Serialize it once.
    put_issued = False

    # Try to issue the put.
    for k in range(3):
        try:
            sn = head_client.execute_command("MEMBER.PUT", i_str, i_str,
                                             _CLIENT_ID)
            put_issued = True
            break
        except redis.exceptions.ConnectionError:
            head_client = RefreshHeadFromMaster(master_client)  # Blocking.
            continue
    if not put_issued:
        raise Exception("Irrecoverable redis connection issue; put client %s" %
                        head_client)

    # Wait for the ack.
    ack = None
    ack_client_okay = False
    for k in range(3):  # Try 3 times.
        try:
            # if k > 0:
            # common.log('k %d pubsub %s' % (k, ack_pubsub.connection))
            # NOTE(zongheng): 1e-4 seems insufficient for an ACK to be
            # delivered back.  1e-3 has the issue of triggering a retry, but
            # then receives an ACK for the old sn (an issue clients will need
            # to address).  Using 10ms for now.
            ack = wait_for_message(ack_pubsub, timeout=1e-2)
            ack_client_okay = True
            break
        except redis.exceptions.ConnectionError as e:
            _, ack_pubsub = RefreshTailFromMaster(master_client,
                                                  _CLIENT_ID)  # Blocking.
            continue

    if not ack_client_okay:
        raise Exception("Irrecoverable redis connection issue; ack client %s" %
                        ack_pubsub.connection)
    elif ack is None:
        # Connected but an ACK was not received after timeout (the update was
        # likely ignored by the store).  Retry.
        fails_since_last_success += 1
        if fails_since_last_success >= max_fails:
            raise Exception(
                "A maximum of %d update attempts have failed; "
                "no acks from the store are received. i = %d, client = %s" %
                (max_fails, i, ack_pubsub.connection))
        _, ack_pubsub = RefreshTailFromMaster(master_client, _CLIENT_ID)
        common.log("%d updates have been ignored since last success, "
                   "retrying Put(%d) with fresh ack client %s" %
                   (fails_since_last_success, i, ack_pubsub.connection))
        time.sleep(3)
        Put(i)
    else:
        # TODO(zongheng): this is a stringent check.  See NOTE above: sometimes
        # we can receive an old ACK.
        assert int(ack["data"]) == sn
        fails_since_last_success = 0
Beispiel #44
0
 def launchChrome(self, url, title):
     action = 'RunPlugin(%s)' % (
         'plugin://plugin.program.chrome.launcher/?kiosk=yes&mode=showSite&stopPlayback=yes&url='
         + url)
     common.log('chrome test:' + str(action))
     xbmc.executebuiltin(action)
Beispiel #45
0
    def run(self, argv=None):

        self.addon = Addon('plugin.video.ViendoKodiStreaming', argv)

        common.log('ViendoKodiStreaming running')

        base = argv[0]
        handle = int(argv[1])
        parameter = argv[2]
        self.base = base
        self.handle = handle

        paramstring = urllib.unquote_plus(parameter)
        common.log(paramstring)

        try:

            # if addon is started
            listItemPath = xbmcUtils.getListItemPath()
            if not listItemPath.startswith(self.base):
                if not ('mode=' in paramstring
                        and not 'mode=1&' in paramstring):
                    xbmcplugin.setPluginFanart(self.handle,
                                               common.Paths.pluginFanart)
                    self.clearCache()

                    #if common.getSetting('autoupdate') == 'true':
                    #    self.update()

            # Main Menu
            if '98VKS' in paramstring:
                nametorrent = os.path.normpath(paramstring.split('url=')[1])
                if nametorrent == 'quasar':
                    addonTorrent = 'item_info_build=plugin://plugin.video.quasar/play?uri=%s'
                elif nametorrent == 'pulsar':
                    addonTorrent = 'item_info_build=plugin://plugin.video.pulsar/play?uri=%s'
                elif nametorrent == 'kmediatorrent':
                    addonTorrent = 'item_info_build=plugin://plugin.video.kmediatorrent/play/%s'
                elif nametorrent == "torrenter":
                    addonTorrent = 'item_info_build=plugin://plugin.video.torrenter/?action=playSTRM&url=%s&not_download_only=True'
                elif nametorrent == "yatp":
                    addonTorrent = 'item_info_build=plugin://plugin.video.yatp/?action=play&torrent=%s'
                else:
                    addonTorrent = 'item_info_build=plugin://plugin.video.xbmctorrent/play/%s'

                cFichero = common.Paths.catchersDir + '/' + 'torrent.txt'
                outfile = open(cFichero, 'w')  # Indicamos el valor 'w'.
                outfile.write('item_info_name=url\n' +
                              'item_info_from=@PARAM1@\n' + addonTorrent +
                              '\n')
                outfile.close()
                common.showInfo(
                    '[COLOR red]NO INSTALAR[/COLOR] conjuntamente los addon pulsar y quasar.\nYa que tendremos problemas de compatibilidad y no funcionará ninguno. [COLOR lime] \nPara los torrent se utilizará: [/COLOR] '
                    + nametorrent)

            elif len(paramstring) <= 2:
                mainMenu = ListItem.create()
                mainMenu['url'] = self.MAIN_MENU_FILE
                tmpList = self.parseView(mainMenu)
                if tmpList:
                    self.currentlist = tmpList

            else:
                [mode, item] = self._parseParameters()

                # switch(mode)
                if mode == Mode.VIEW:
                    tmpList = self.parseView(item)
                    if tmpList:
                        self.currentlist = tmpList
                        count = len(self.currentlist.items)
                        if count == 1:
                            # Autoplay single video
                            autoplayEnabled = common.getSetting(
                                'autoplay') == 'true'
                            if autoplayEnabled:
                                videos = self.currentlist.getVideos()
                                if len(videos) == 1:
                                    self.playVideo(videos[0], True)

                elif mode == Mode.ADDITEM:
                    tmp = os.path.normpath(paramstring.split('url=')[1])
                    if tmp:
                        suffix = tmp.split(os.path.sep)[-1]
                        tmp = tmp.replace(suffix,
                                          '') + urllib.quote_plus(suffix)
                    if self.favouritesManager.add(tmp):
                        xbmc.executebuiltin('Container.Refresh()')

                elif mode in [
                        Mode.ADDTOFAVOURITES, Mode.REMOVEFROMFAVOURITES,
                        Mode.EDITITEM
                ]:

                    if mode == Mode.ADDTOFAVOURITES:
                        self.favouritesManager.addItem(item)
                    elif mode == Mode.REMOVEFROMFAVOURITES:
                        self.favouritesManager.removeItem(item)
                        xbmc.executebuiltin('Container.Refresh()')
                    elif mode == Mode.EDITITEM:
                        if self.favouritesManager.editItem(item):
                            xbmc.executebuiltin('Container.Refresh()')

                elif mode == Mode.EXECUTE:
                    self.executeItem(item)

                elif mode == Mode.PLAY:
                    self.playVideo(item)

                elif mode == Mode.WEBDRIVER:
                    url = urllib.quote(item['url'])
                    title = item['title']
                    self.playWebDriver(url, title)

                elif mode == Mode.QUEUE:
                    self.queueAllVideos(item)

                elif mode == Mode.CHROME:
                    url = urllib.quote(item['url'])
                    title = item['title']
                    self.launchChrome(url, title)

                elif mode == Mode.SAY:
                    #title = item['title']
                    url = ""

                elif mode == Mode.INSTALLADDON:
                    success = install(item['url'])
                    if success:
                        xbmc.sleep(100)
                        if xbmcUtils.getCurrentWindowXmlFile(
                        ) == 'DialogAddonSettings.xml':
                            # workaround to update settings dialog
                            common.setSetting('', '')

        except Exception, e:
            common.showError('Error running ViendoKodiStreaming')
            common.log('Error running ViendoKodiStreaming. Reason:' + str(e))
Beispiel #46
0
    def createXBMCListItem(self, item):
        title = item['title']

        m_type = item['type']

        icon = item['icon']

        if icon and not icon.startswith('http'):
            try:
                if not fu.fileExists(icon):
                    tryFile = os.path.join(common.Paths.modulesDir, icon)
                    if not fu.fileExists(tryFile):
                        tryFile = os.path.join(common.Paths.customModulesDir,
                                               icon)
                    if fu.fileExists(tryFile):
                        icon = tryFile
            except:
                pass

        if not icon:
            if m_type == 'video':
                icon = common.Paths.defaultVideoIcon
            else:
                icon = common.Paths.defaultCategoryIcon

        fanart = item['fanart']
        if not fanart:
            fanart = common.Paths.pluginFanart

        liz = xbmcgui.ListItem(title)
        try:
            liz.setArt({'thumb': icon, 'fanart': fanart})
        except:
            liz.setProperty('fanart_image', fanart)
            liz.setThumbnailImage(icon)
            common.log('main.py:374: setThumbnailImage is deprecated')
        """
        General Values that apply to all types:
            count         : integer (12) - can be used to store an id for later, or for sorting purposes
            size          : long (1024) - size in bytes
            date          : string (%d.%m.%Y / 01.01.2009) - file date

        Video Values:
            genre         : string (Comedy)
            year          : integer (2009)
            episode       : integer (4)
            season        : integer (1)
            top250        : integer (192)
            tracknumber   : integer (3)
            rating        : float (6.4) - range is 0..10
            watched       : depreciated - use playcount instead
            playcount     : integer (2) - number of times this item has been played
            overlay       : integer (2) - range is 0..8.  See GUIListItem.h for values
            cast          : list (Michal C. Hall)
            castandrole   : list (Michael C. Hall|Dexter)
            director      : string (Dagur Kari)
            mpaa          : string (PG-13)
            plot          : string (Long Description)
            plotoutline   : string (Short Description)
            title         : string (Big Fan)
            originaltitle : string (Big Fan)
            duration      : string (3:18)
            studio        : string (Warner Bros.)
            tagline       : string (An awesome movie) - short description of movie
            writer        : string (Robert D. Siegel)
            tvshowtitle   : string (Heroes)
            premiered     : string (2005-03-04)
            status        : string (Continuing) - status of a TVshow
            code          : string (tt0110293) - IMDb code
            aired         : string (2008-12-07)
            credits       : string (Andy Kaufman) - writing credits
            lastplayed    : string (%Y-%m-%d %h:%m:%s = 2009-04-05 23:16:04)
            album         : string (The Joshua Tree)
            votes         : string (12345 votes)
            trailer       : string (/home/user/trailer.avi)
        """

        infoLabels = {}
        for video_info_name in item.infos.keys():
            infoLabels[video_info_name] = item[video_info_name]
        infoLabels['title'] = title

        liz.setInfo('video', infoLabels)

        url = urllib.unquote_plus(item['url'])
        liz.setPath(url)

        if m_type == 'video':
            liz.setProperty('IsPlayable', 'true')

        if title.startswith('p2pcast'):
            try:
                liz.setMimeType('application/vnd.apple.mpegurl')
                #liz.setContentLookup(False)
            except:
                common.showError('Update Kodi to 16+ to view this stream')
                return None

        return liz
Beispiel #47
0
    def __parseCommands(self, item, src, convCommands):
        #common.log('_parseCommands called')
        # helping function
        def parseCommand(txt):
            command = {"command": txt, "params": ""}
            if txt.find("(") > -1:
                command["command"] = txt[0:txt.find("(")]
                command["params"] = txt[len(command["command"]) + 1:-1]
            return command

        try:
            src = src.encode('utf-8')
        except:
            pass
        for convCommand in convCommands:
            pComm = parseCommand(convCommand)
            command = pComm["command"]
            params = pComm["params"]

            if params.find('@REFERER@'):
                referer = item['referer']
                if not referer:
                    referer = ''
                params = params.replace('@REFERER@', referer)

            if command == 'convDate':
                src = cc.convDate(params, src)

            elif command == 'convTimestamp':
                src = cc.convTimestamp(params, src)

            elif command == 'select':
                src = cc.select(params, src)
                if not src:
                    continue

            elif command == 'smart_unicode':
                src = enc.smart_unicode(params.strip("'").replace('%s', src))

            elif command == 'safeGerman':
                src = enc.safeGerman(src)

            elif command == 'safeRegex':
                src = enc.safeRegexEncoding(
                    params.strip("'").replace('%s', enc.smart_unicode(src)))

            elif command == 'replaceFromDict':
                dictName = str(params.strip('\''))
                path = os.path.join(common.Paths.dictsDir, dictName + '.txt')
                if not (os.path.exists(path)):
                    common.log('Dictionary file not found: ' + path)
                    continue
                src = cc.replaceFromDict(path, src)

            elif command == 'time':
                src = time.time()

            elif command == 'timediff':
                src = dt.timediff(src, params.strip('\''))

            elif command == 'offset':
                src = cc.offset(params, src)

            elif command == 'getSource':
                src = cc.getSource(params, src)

            elif command == 'getRedirect':
                src = get_redirected_url(params.strip("'").replace('%s', src))

            elif command == 'quote':
                try:
                    src = urllib.quote(
                        params.strip("'").replace('%s', urllib.quote(src)))
                except:
                    cleanParams = params.strip("'")
                    cleanParams = cleanParams.replace("%s",
                                                      src.encode('utf-8'))
                    src = urllib.quote(cleanParams)

            elif command == 'unquote':
                src = urllib.unquote(params.strip("'").replace('%s', src))

            elif command == 'parseText':
                src = cc.parseText(item, params, src)

            elif command == 'getInfo':
                src = cc.getInfo(item, params, src)

            elif command == 'decodeBase64':
                src = cc.decodeBase64(src)

            elif command == 'decodeRawUnicode':
                src = cc.decodeRawUnicode(src)

            elif command == 'replace':
                src = cc.replace(params, src)

            elif command == 'replaceRegex':
                src = cc.replaceRegex(params, src)

            elif command == 'ifEmpty':
                src = cc.ifEmpty(item, params, src)

            elif command == 'isEqual':
                src = cc.isEqual(item, params, src)

            elif command == 'ifFileExists':
                src = cc.ifFileExists(item, params, src)

            elif command == 'ifExists':
                src = cc.ifExists(item, params, src)

            elif command == 'encryptJimey':
                src = crypt.encryptJimey(params.strip("'").replace('%s', src))

            elif command == 'destreamer':
                src = crypt.destreamer(params.strip("'").replace('%s', src))

            elif command == 'unixTimestamp':
                src = dt.getUnixTimestamp()

            elif command == 'urlMerge':
                src = cc.urlMerge(params, src)

            elif command == 'translate':
                try:
                    src = common.translate(int(src))
                except:
                    pass

            elif command == 'camelcase':
                src = enc.smart_unicode(src)
                src = string.capwords(string.capwords(src, '-'))

            elif command == 'demystify':
                print 'demystify'
                src = crypt.doDemystify(src)
                print 'after demystify', src

            elif command == 'random':
                paramArr = params.split(',')
                minimum = int(paramArr[0])
                maximum = int(paramArr[1])
                src = str(random.randrange(minimum, maximum))

            elif command == 'debug':
                common.log('Debug from cfg file: ' + src)

            elif command == 'divide':
                paramArr = params.split(',')
                a = paramArr[0].strip().strip("'").replace('%s', src)
                a = resolveVariable(a, item)
                b = paramArr[1].strip().strip("'").replace('%s', src)
                b = resolveVariable(b, item)

                if not a or not b:
                    continue

                a = int(a)
                b = int(b)
                try:
                    src = str(a / b)
                except:
                    pass

        return src
Beispiel #48
0
    def findGames(self, orgGames, games):
        xbmc.executebuiltin("Dialog.Close(busydialog)")
        isCanceled = False
        try:
            steamPath = self.getSteamPath()
            if (steamPath):
                steamAppPath = os.path.join(os.path.abspath(steamPath),'steamapps')

                files = glob('{0}/*.acf'.format(steamAppPath))
                numFiles = len(files)
                if (numFiles > 0):
                    counter = 0

                    pDialog = xbmcgui.DialogProgress()
                    pDialog.create(__addonname__, localize(33003))
                    pDialog.update(0, localize(33003))

                    for filePath in glob('{0}/*.acf'.format(steamAppPath)):
                        log('findGames()::filepath -> %s' % filePath)
                        #try:
                        gameDict = parse_acf(filePath)

                        if (gameDict['appstate']['bytestodownload'] != '0' and gameDict['appstate']['bytestodownload'] == gameDict['appstate']['bytesdownloaded']):

                            orgGame = None
                            gameId = gameDict['appstate']['appid']
                            for game in orgGames:
                                if (game.gameId == gameId):
                                    orgGame = game
                                    break;

                            name = gameDict['appstate']['name']
                            path = ''
                            isNameChanged = 0
                            isPathChanged = 0
                            isIconChanged = 0
                            isFanartChanged = 0
                            thumbImage = ''
                            fanartImage = ''
                            type = 1

                            if ((not orgGame) or (orgGame is not None and orgGame.isFanartChanged == 0 and orgGame.isIconChanged == 0)):
                                gameFiles = self.downloadGameImage(gameId,gameDict['appstate']['name'])
                                thumbImage = gameFiles[0]
                                fanartImage = gameFiles[1]
                            elif (orgGame is not None and orgGame.isFanartChanged == 0):
                                gameFiles = self.downloadGameImage(gameId,gameDict['appstate']['name'])
                                thumbImage = orgGame.thumbImage
                                fanartImage = gameFiles[1]
                                isIconChanged = 1
                            elif (orgGame is not None and orgGame.isIconChanged == 0):
                                gameFiles = self.downloadGameImage(gameId,gameDict['appstate']['name'])
                                thumbImage = gameFiles[0]
                                fanartImage = orgGame.fanartImage
                                isFanartChanged = 1
                            else:
                                thumbImage = orgGame.thumbImage
                                fanartImage = orgGame.fanartImage
                                isIconChanged = 1
                                isFanartChanged = 1

                            if (orgGame is not None and orgGame.isNameChanged == 1):
                                name = orgGame.title
                                isNameChanged = 1

                            if (orgGame is not None and orgGame.isPathChanged == 1):
                                name = orgGame.path
                                isPathChanged = 1

                            games[gameId] = {}
                            games[gameId]['name'] = name
                            games[gameId]['path'] = path
                            games[gameId]['isNameChanged'] = isNameChanged
                            games[gameId]['isPathChanged'] = isPathChanged
                            games[gameId]['isIconChanged'] = isIconChanged
                            games[gameId]['isFanartChanged'] = isFanartChanged
                            games[gameId]['type'] = type
                            games[gameId]['thumbImage'] = thumbImage
                            games[gameId]['fanartImage'] = fanartImage

                            counter = counter + 1
                        #except:
                            #log('unable to parse the file %s' % filePath)

                        if (pDialog.iscanceled()):
                            isCanceled = True
                            break;

                        pDialog.update(int((float(counter)/float(numFiles))*100), localize(33003))

                    pDialog.close()
        except Exception,e:
            log(e.message)
            log(e.__class__.__name__)
            traceback.print_exc(e)
Beispiel #49
0
def main():
    global arguments
    global grep_list
    global yara_rules
    parser = ArgumentParser()
    parser.add_argument(
        "-D",
        dest="download",
        required=False,
        action="store_true",
        default=False,
        help="Download files. This requires significant disk space.")
    parser.add_argument(
        "-d",
        dest="savedir",
        required=False,
        default='',
        help=
        "If -D, then -d 1 to create save directories for each bucket with results."
    )
    parser.add_argument("-l", dest="hostlist", required=True, help="")
    parser.add_argument("-g",
                        dest="grepwords",
                        required=False,
                        help="Provide a wordlist to grep for.")
    parser.add_argument("-m",
                        dest="maxsize",
                        type=int,
                        required=False,
                        default=1024,
                        help="Maximum file size to download.")
    parser.add_argument("-t",
                        dest="threads",
                        type=int,
                        required=False,
                        default=1,
                        help="Number of threads.")
    parser.add_argument(
        "-Y",
        dest="runyara",
        required=False,
        action="store_true",
        default=False,
        help="Run Yara rules against downloads and only save matches")
    if len(sys.argv) == 1:
        print_banner()
        parser.error("No arguments given.")
        parser.print_usage
        sys.exit()

    # output parsed arguments into a usable object
    arguments = parser.parse_args()

    if arguments.runyara != False and ENABLE_YARA == True:
        yara_index_build()
        yara_rules = yara.compile("YaraRules/index.yar")
    # specify primary variables
    if arguments.grepwords != None:
        with open(arguments.grepwords, "r") as grep_file:
            grep_content = grep_file.readlines()
        grep_list = [g.strip() for g in grep_content]

    if arguments.download and arguments.savedir:
        print(
            "Downloads enabled (-D), save directories (-d) for each host will be created/used."
        )
        print("Default Download Directory: {}".format(arguments.savedir))
    elif arguments.download and not arguments.savedir:
        print("Downloads enabled (-D), will be saved to current directory.")
    else:
        print("Downloads were not enabled (-D), not saving results locally.")
    # start up bucket workers
    for i in range(0, arguments.threads):
        log('Starting thread...', logging.DEBUG)
        t = Thread(target=bucket_worker)
        t.daemon = True
        t.start()

    # start download workers
    for i in range(1, arguments.threads):
        t = Thread(target=downloadWorker)
        t.daemon = True
        t.start()

    with open(arguments.hostlist) as f:
        for line in f:
            bucket = 'http://' + line.rstrip() + '.s3.amazonaws.com'
            log('Queuing: {}'.format(bucket))
            bucket_q.put(bucket)

    bucket_q.join()
    if arguments.download:
        download_q.join()

    cleanUp()
Beispiel #50
0
    def downloadGameImage(self,gameId, gameName):
        log('Downloading Steam game images for {0} -> {1}'.format(gameId, gameName))
    
        files = []

        valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
        safeGameName = ''.join(c for c in gameName if c in valid_chars)

        thumbFilePath = os.path.join(self.defaultThumbPath,'thumb_{0}_{1}.jpg'.format(safeGameName, int((time.mktime(datetime.datetime.now().timetuple())))))
        fanartPath = os.path.join(self.defaultFanartPath,'fanart_{0}_{1}.jpg'.format(safeGameName, int((time.mktime(datetime.datetime.now().timetuple())))))

        isFanartFound = False
        isThumbImageFound = False

        for fileName in glob(os.path.join(self.defaultThumbPath,'thumb_{0}*.*'.format(safeGameName))):
            thumbFilePath = fileName
            isThumbImageFound = True

        for fileName in glob(os.path.join(self.defaultFanartPath,'fanart_{0}*.*'.format(safeGameName))):
            fanartPath = fileName
            isFanartFound = True

        try:
            if (not isFanartFound):
                gameUrlQuery = {'exactname':gameName.encode('utf8')}

                requestURL = 'http://thegamesdb.net/api/GetGame.php?' + urllib.urlencode(gameUrlQuery)
    
                opener = urllib2.build_opener()
                opener.addheaders = [('User-agent', 'Mozilla/5.0')]

                xmlResponse = opener.open(requestURL, timeout=30)

                strXml = xmlResponse.read()

                root = ET.fromstring(strXml)
                baseImgUrl = root[0].text
                fanartNodes = root.findall('.//Game/Images/fanart/original')
                fanartImg = fanartNodes[0].text

                downloadFile(baseImgUrl + fanartImg, fanartPath)
                isFanartFound = True
        except:
            pass

        try:
            if (not isThumbImageFound or not isFanartFound):
                response = urllib.urlopen('http://store.steampowered.com/api/appdetails?appids={0}'.format(gameId))
                gameJason = json.loads(response.read())
                response.close()

                if(not isThumbImageFound):
                    downloadFile(gameJason[gameId]['data']['header_image'], thumbFilePath)
                    isThumbImageFound = True

                if(not isFanartFound):
                    downloadFile(gameJason[gameId]['data']['screenshots'][0]['path_full'], fanartPath)
                    isFanartFound = True    
        except:
            pass    

        if(not isThumbImageFound):
            thumbFilePath = os.path.join(__addonpath__,'resources','skins','Default','media','alienware','steam.png')
        if(not isFanartFound):
            fanartPath = os.path.join(__addonpath__,'resources','skins','Default','media','alienware','fanart_steam.png')

        files.append(thumbFilePath)
        files.append(fanartPath)
    
        return files
Beispiel #51
0
import os
from common import log

port = os.environ.get('PORT', 8000)

app_id = ""
app_secret = ""

try:
    with open("skypetoken", "r") as f:
        content = f.read().splitlines()

    try:
        app_id = content[0]
        app_secret = content[1]
    except:
        log("skypetoken Empty")
except:
    log("skypetoken not found getting tokens from envvar")

app_id = os.environ.get('app_id', app_id)
app_secret = os.environ.get('app_secret', app_secret)
Beispiel #52
0
    def run(self, argv=None):
        self.addon = Addon('plugin.video.SportsDevil', argv)
        common.log('SportsDevil running')
        
        base = argv[0]
        handle = int(argv[1])
        parameter = argv[2]
        self.base = base
        self.handle = handle
        
        paramstring = urllib.unquote_plus(parameter)
        common.log(paramstring)
        
        try:
            
            # if addon is started
            listItemPath = xbmcUtils.getListItemPath()
            if not listItemPath.startswith(self.base):
                if not('mode=' in paramstring and not 'mode=1&' in paramstring):   
                    xbmcplugin.setPluginFanart(self.handle, common.Paths.pluginFanart)
                    self.clearCache()
            
            # Main Menu
            if len(paramstring) <= 2:                
                mainMenu = ListItem.create()
                mainMenu['url'] = self.MAIN_MENU_FILE
                tmpList = self.parseView(mainMenu)
                if tmpList:
                    self.currentlist = tmpList
                
            else:
                [mode, item] = self._parseParameters()
                # switch(mode)
                if mode == Mode.VIEW:
                    tmpList = self.parseView(item)
                    if tmpList:
                        self.currentlist = tmpList
                        count = len(self.currentlist.items)
                        if count == 1:
                            # Autoplay single video
                            autoplayEnabled = common.getSetting('autoplay') == 'true'
                            if autoplayEnabled:
                                videos = self.currentlist.getVideos()
                                if len(videos) == 1:
                                    self.playVideo(videos[0], True)
                                    


                elif mode == Mode.ADDITEM:
                    
                    tmp = os.path.normpath(paramstring.split('url=')[1])
                    if tmp:
                        suffix = tmp.split(os.path.sep)[-1]
                        tmp = tmp.replace(suffix,'') + urllib.quote_plus(suffix)
                        
                    if self.favouritesManager.add(tmp):
                        xbmc.executebuiltin('Container.Refresh()')


                elif mode in [Mode.ADDTOFAVOURITES, Mode.REMOVEFROMFAVOURITES, Mode.EDITITEM]:

                    if mode == Mode.ADDTOFAVOURITES:
                        self.favouritesManager.addItem(item)
                    elif mode == Mode.REMOVEFROMFAVOURITES:
                        self.favouritesManager.removeItem(item)
                        xbmc.executebuiltin('Container.Refresh()')
                    elif mode == Mode.EDITITEM:
                        if self.favouritesManager.editItem(item):
                            xbmc.executebuiltin('Container.Refresh()')


                elif mode == Mode.EXECUTE:
                    self.executeItem(item)

                elif mode == Mode.PLAY:
                    self.playVideo(item)
                
                elif mode == Mode.SLPROXY:
                    self.playSLProxy(item)
                
                elif mode == Mode.WEBDRIVER:
                    url = urllib.quote(item['url'])
                    title = item['title']
                    self.playWebDriver(url, title)

                elif mode == Mode.QUEUE:
                    self.queueAllVideos(item)

                elif mode == Mode.CHROME:
                    url = urllib.quote(item['url'])
                    title = item['title']
                    self.launchChrome(url, title)
                
                elif mode == Mode.INSTALLADDON:
                    success = install(item['url'])
                    if success:
                        xbmc.sleep(100)
                        if xbmcUtils.getCurrentWindowXmlFile() == 'DialogAddonSettings.xml':
                            # workaround to update settings dialog
                            common.setSetting('', '')
                            

        except Exception, e:
            common.showError('Error running SportsDevil')
            common.log('Error running SportsDevil. Reason:' + str(e))
Beispiel #53
0
def status403(line):
    log("403 Error: " + line.rstrip() + " is not accessible.")
Beispiel #54
0
 def say_hall(self, data):
     log(data)
     message = data.get('message', None)
     say(self.client, message, self.clients, 'hall.message')
Beispiel #55
0
def cleanUp():
    log("Cleaning up files...", logging.DEBUG)
Beispiel #56
0
 def delete_networks(self):
     for net in self.net_names:
         log('Deleting network %s' % net)
         exec_cmd('virsh net-destroy %s' % net, False)
         exec_cmd('virsh net-undefine %s' % net, False)
Beispiel #57
0
def queue_up_download(filepath):
    download_q.put(filepath)
    log('Collectable: {}'.format(filepath), logging.DEBUG)
    write_interesting_file(filepath)
Beispiel #58
0
    def parseView(self, lItem):
        def endOfDirectory(succeeded=True):
            xbmcplugin.endOfDirectory(handle=self.handle,
                                      succeeded=succeeded,
                                      cacheToDisc=True)

        if not lItem:
            endOfDirectory(False)
            return None

        if lItem['type'] == 'search':
            search_phrase = self.getSearchPhrase()
            if not search_phrase:
                common.log("search canceled")
                endOfDirectory(False)
                return None
            else:
                lItem['type'] = 'rss'
                lItem['url'] = lItem['url'] % (
                    urllib.quote_plus(search_phrase))

        url = lItem['url']

        if url == common.Paths.customModulesFile:
            self.customModulesManager.getCustomModules()

        tmpList = None
        result = self.parser.parse(lItem)
        if result.code == ParsingResult.Code.SUCCESS:
            tmpList = result.list
        else:
            if result.code == ParsingResult.Code.CFGFILE_NOT_FOUND:
                common.showError("Cfg file not found")
            elif result.code == ParsingResult.Code.CFGSYNTAX_INVALID:
                common.showError("Cfg syntax invalid")
            elif result.code == ParsingResult.Code.WEBREQUEST_FAILED:
                common.showError("Web request failed")

            endOfDirectory(False)
            return None

        # if it's the main menu, add folder 'Favourites' and 'Custom Modules
        if url == self.MAIN_MENU_FILE:
            tmp = ListItem.create()
            tmp['title'] = 'Favourites'
            tmp['type'] = 'rss'
            tmp['icon'] = os.path.join(common.Paths.imgDir, 'bookmark.png')
            tmp['url'] = str(common.Paths.favouritesFile)
            tmpList.items.insert(0, tmp)

            tmp = ListItem.create()
            tmp['title'] = '[COLOR red]Custom Modules[/COLOR]'
            tmp['type'] = 'rss'
            tmp['url'] = os.path.join(common.Paths.customModulesDir,
                                      'custom.cfg')
            tmpList.items.insert(0, tmp)

        # if it's the favourites menu, add item 'Add item'
        elif url == common.Paths.favouritesFile or url.startswith(
                'favfolders'):

            if url.startswith('favfolders'):
                url = os.path.normpath(
                    os.path.join(common.Paths.favouritesFolder, url))

            tmp = ListItem.create()
            tmp['title'] = 'Add item...'
            tmp['type'] = 'command'
            tmp['icon'] = os.path.join(common.Paths.imgDir, 'bookmark_add.png')
            action = 'RunPlugin(%s)' % (self.base + '?mode=' +
                                        str(Mode.ADDITEM) + '&url=' + url)
            tmp['url'] = action
            tmpList.items.append(tmp)

        # if it's the custom modules  menu, add item 'more...'
        elif url == common.Paths.customModulesFile:
            tmp = ListItem.create()
            tmp['title'] = 'more...'
            tmp['type'] = 'command'
            #tmp['icon'] = os.path.join(common.Paths.imgDir, 'bookmark_add.png')
            action = 'RunPlugin(%s)' % (self.base + '?mode=' + str(
                Mode.DOWNLOADCUSTOMMODULE) + '&url=')
            tmp['url'] = action
            tmpList.items.append(tmp)

        # Create menu or play, if it's a single video and autoplay is enabled
        proceed = False

        count = len(tmpList.items)
        if count == 0:
            if url.startswith('favfolders'):
                proceed = True
            else:
                common.showInfo('No stream available')
        elif count > 0 and not (common.getSetting('autoplay') == 'true' and
                                count == 1 and len(tmpList.getVideos()) == 1):
            # sort methods
            sortKeys = tmpList.sort.split('|')
            setSortMethodsForCurrentXBMCList(self.handle, sortKeys)

            # Add items to XBMC list
            for m in tmpList.items:
                self.addListItem(m, len(tmpList.items))

            proceed = True

        endOfDirectory(proceed)
        common.log('End of directory')
        return tmpList
Beispiel #59
0
 def get(self, url=None):
     if url is not None:
         log("get:" + url)
     m = self.get_argument("m")
     p = json.loads(self.get_argument("p", "{}"))
     return self.process(m, p)
Beispiel #60
0
 def endOfDirectory(succeeded=True):
     if self.handle > -1:
         xbmcplugin.endOfDirectory(handle=self.handle, succeeded=succeeded, cacheToDisc=True)
     else:
         common.log('Handle -1')