Ejemplo n.º 1
0
def old_login(username, md5_password):
    from api import agent_header
    exponent = int("010001", 16)
    modulus = int("AC69F5CCC8BDE47CD3D371603748378C9CFAD2938A6B021E0E191013975AD683F5CBF9ADE8BD7D46B4D2EC2D78A"
                  "F146F1DD2D50DC51446BB8880B8CE88D476694DFC60594393BEEFAA16F5DBCEBE22F89D640F5336E42F587DC4AF"
                  "EDEFEAC36CF007009CCCE5C1ACB4FF06FBA69802A8085C2C54BADD0597FC83E6870F1E36FD", 16)

    param = '{"cmdID":1,"isCompressed":0,"rsaKey":{"n":"AC69F5CCC8BDE47CD3D371603748378C9CFAD2938A6B0' \
            '21E0E191013975AD683F5CBF9ADE8BD7D46B4D2EC2D78AF146F1DD2D50DC51446BB8880B8CE88D476694DFC60594393BEEFAA16F' \
            '5DBCEBE22F89D640F5336E42F587DC4AFEDEFEAC36CF007009CCCE5C1ACB4FF06FBA69802A8085C2C54BADD0597FC83E6870F1E3' \
            '6FD","e":"010001"},"businessType":%s,"passWord":"******","loginType":0,"sdkVersion":177588,' \
            '"appName":"ANDROID-com.xunlei.redcrystalandroid","platformVersion":1,"devicesign":"%s",' \
            '"sessionID":"","protocolVersion":%s,"userName":"******","extensionList":"","sequenceNo":%s,' \
            '"peerID":"","clientVersion":"1.0.0"}'

    _chars = "0123456789ABCDEF"

    deviceid = username
    device_id = md5(deviceid)

    appName = 'com.xunlei.redcrystalandroid'
    businessType = '61'
    key = 'C2049664-1E4A-4E1C-A475-977F0E207C9C'
    key_md5 = md5(key)

    device_sign = "div100.%s%s" % (device_id, md5(sha1("%s%s%s%s" % (device_id, appName, businessType, key_md5))))

    hash_password = hex(pow_mod(StrToInt(md5_password), exponent, modulus))[2:].upper().zfill(256)

    params = param % (61, hash_password, device_sign, 108, username, 1000006)

    r = requests.post("https://login.mobile.reg2t.sandai.net/", data=params, headers=agent_header, verify=False)
    login_status = json.loads(r.text)

    return login_status
Ejemplo n.º 2
0
def continuity(url):
    from util import md5
    format = '%25s: %s'

    # first fetch the file with the normal http handler
    opener = urllib2.build_opener()
    urllib2.install_opener(opener)
    fo = urllib2.urlopen(url)
    foo = fo.read()
    fo.close()
    m = md5(foo)
    print format % ('normal urllib', m.hexdigest())

    # now install the keepalive handler and try again
    opener = urllib2.build_opener(HTTPHandler())
    urllib2.install_opener(opener)

    fo = urllib2.urlopen(url)
    foo = fo.read()
    fo.close()
    m = md5(foo)
    print format % ('keepalive read', m.hexdigest())

    fo = urllib2.urlopen(url)
    foo = ''
    while True:
        f = fo.readline()
        if f:
            foo = foo + f
        else: break
    fo.close()
    m = md5(foo)
    print format % ('keepalive readline', m.hexdigest())
Ejemplo n.º 3
0
def old_login(username, md5_password):

    hash_password = rsa_encode(md5_password)

    _chars = "0123456789ABCDEF"
    peer_id = ''.join(random.sample(_chars, 16))

    device_id = md5("%s23333" % md5_password)  # just generate a 32bit string

    appName = 'com.xunlei.redcrystalandroid'
    key = 'C2049664-1E4A-4E1C-A475-977F0E207C9C'
    md5_key = md5(key)

    device_sign = 'div100.%s%s' % (
        device_id, md5(sha1("%s%s%s%s" % (device_id, appName, 61, md5_key))))

    payload = json.dumps({
        "protocolVersion": PROTOCOL_VERSION,
        "sequenceNo": 1000001,
        "platformVersion": 1,
        "sdkVersion": 177588,
        "peerID": peer_id,
        "businessType": 61,
        "clientVersion": APP_VERSION,
        "devicesign": device_sign,
        "isCompressed": 0,
        "cmdID": 1,
        "userName": username,
        "passWord": hash_password,
        "loginType": 0,
        "sessionID": "",
        "verifyKey": "",
        "verifyCode": "",
        "appName": "ANDROID-com.xunlei.redcrystalandroid",
        "rsaKey": {
            "e": "%06X" % rsa_pubexp,
            "n": long2hex(rsa_mod)
        },
        "extensionList": ""
    })

    headers = {
        'User-Agent':
        "Mozilla/5.0 (iPhone; CPU iPhone OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Mobile/9B176 MicroMessenger/4.3.2"
    }
    r = requests.post("https://login.mobile.reg2t.sandai.net/",
                      data=payload,
                      headers=headers,
                      verify=False)

    login_status = json.loads(r.text)

    return login_status
def collect_tweets_by_search_terms(search_configs_filepath, output_folder,
                                   config):

    apikeys = list(config['apikeys'].values()).pop()

    search_configs = {}
    with open(os.path.abspath(search_configs_filepath),
              'r') as search_configs_rf:
        search_configs = json.load(search_configs_rf)

    for search_config_id in itertools.cycle(search_configs):

        search_config = search_configs[search_config_id]

        search_terms = [term.lower() for term in search_config['terms']]
        querystring = '%s' % (' OR '.join('(' + term + ')'
                                          for term in search_terms))
        since_id = search_config[
            'since_id'] if 'since_id' in search_config else 0
        geocode = tuple(search_config['geocode']) if (
            'geocode' in search_config and search_config['geocode']) else None

        logger.info(
            'REQUEST -> (md5(querystring): [%s]; since_id: [%d]; geocode: [%s])'
            % (util.md5(querystring.encode('utf-8')), since_id, geocode))

        try:
            twitterCralwer = TwitterCrawler(apikeys=apikeys,
                                            client_args=CLIENT_ARGS,
                                            output_folder=output_folder)
            since_id = twitterCralwer.search_by_query(querystring,
                                                      geocode=geocode,
                                                      since_id=since_id)
        except Exception as exc:
            logger.error(exc)
            logger.error(util.full_stack())
            pass

        search_config['since_id'] = since_id
        search_config['querystring'] = querystring
        search_config['geocode'] = geocode

        search_configs[search_config_id] = search_config

        flash_cmd_config(search_configs, search_configs_filepath,
                         output_folder)

        logger.info(
            'COMPLETED -> (md5(querystring): [%s]; since_id: [%d]; geocode: [%s])'
            % (util.md5(querystring.encode('utf-8')), since_id, geocode))
        logger.info('PAUSE %ds to CONTINUE...' % WAIT_TIME)
        time.sleep(WAIT_TIME)
Ejemplo n.º 5
0
def upload_image():
    if 'file' not in request.files:
        raise HttpError(400, "缺少file参数")
    file = request.files['file']
    if file.filename == '':
        raise HttpError(400, "未发现文件")
    content = file.read()

    # 图片格式限制
    extname = file.filename.rsplit('.', 1)[1].lower()
    if extname not in ['png', 'jpg', 'jpeg', 'gif']:
        raise HttpError(400, "不支持此格式图片上传")

    # 图片大小限制
    max_size = 5
    size = int(math.ceil(len(content) / 1024.0))  # size KB
    if size > max_size * 1024:
        raise HttpError(400, u"图片不能超过%sM" % max_size)

    md5sum = md5(content).lower()
    full_path = "%s/%s/%s/%s.%s" % (app.config["IMAGE_UPLOAD_PATH"],
                                    md5sum[:2], md5sum[2:4], md5sum, extname)

    directory = os.path.dirname(full_path)
    if not os.path.exists(directory):
        os.makedirs(directory)

    output_file = open(full_path, 'w+')
    output_file.write(content)

    url = path_to_url(full_path)
    res = rpc.invbox.check_add_image(md5sum,
                                     url,
                                     base_url=app.config["DOMAIN"])
    return jsonify(res)
Ejemplo n.º 6
0
	def get_page_comments(self,shortcode,delay=DELAY,count=-1,save=False,path=None,tname=None):
		results = []
		_count = 0
		page = self.get_page_info(shortcode)
		comment_card = page['graphql']['shortcode_media']['edge_media_to_comment']
		total = comment_card['count']
		page_info = comment_card['page_info']
		top_comments = comment_card['edges']
		end_cursor = page_info['end_cursor']
		has_next = page_info['has_next_page']
		headers = COMMON_HEADERS
		headers['x-ig-app-id']=self.app_id
		headers['referer'] = API_PICTURE_PAGE.format(shortcode=shortcode)
		_check = count if count > 0 else total
		for i in top_comments:
			if save:
				self.db.save(i,tname=tname)
			results.append(i)
			_count += 1
			if (_count >= count or _count >= total) and (count > 0):
				logger.info(f'[Done]Get crawled comments of page:"{shortcode}":{len(results)}.[Total({total})]')
				return results
		if not has_next:
			logger.info(f'[Done]Get crawled comments of page:"{shortcode}":{len(results)}.[Total({total})]')
			return results
		while 1:
			if not end_cursor:
				logger.info(f'[Done]Get crawled comments of page:"{shortcode}":{len(results)}.[Total({total})]')
				break
			params = copy.deepcopy(COMMENTS_PARAMS)
			params['query_hash']=self.comment_hash
			params['variables']=params['variables'].replace('$',end_cursor).replace('%',shortcode)
			md5ed = md5(self.rhx_gis + ":" + params['variables'])
			headers['x-instagram-gis']=md5ed
			response = send_request(API_USER_POSTS,
				params=params,
				headers=headers,
				delay=delay,
				proxy=PROXY_GLOBAL,
				json=True)
			json_data = response.json()
			data = json_data['data']['shortcode_media']['edge_media_to_comment']['edges']
			page_info = json_data['data']['shortcode_media']['edge_media_to_comment']['page_info']
			for i in data:
				if save:
					self.db.save(i,tname=tname)
				results.append(i)
				_count += 1
				if (_count >= count or _count >= total) and (count > 0):
					logger.info(f'[Done]Get crawled comments of page:"{shortcode}"'
						f':{len(results)}.[Total({total})]')
					return results
			logger.info(f'Current crawled comments of page "{shortcode}"'
				f':{len(results)}.[{round(len(results)/_check,4)*100  if _check else 0}%]')
			end_cursor = page_info['end_cursor']
			if not page_info['has_next_page']:
				logger.info(f'[Done]Get crawled comments of page:"{shortcode}"'
					f':{len(results)}.[Total({total})]')
				break
		return results
Ejemplo n.º 7
0
 def _hash(git_url: str) -> str:
     """
     hash the git url as a unique key
     :param git_url: git url
     :return: md5 hash of git url
     """
     return util.md5(git_url)
Ejemplo n.º 8
0
def init_base():
    users = {
        "luke": "13064754229",
        "april": "13621713595",
    }

    for name, mobile in users.items():
        admin = M.Admin.create(
            mobile=mobile,
            username=name,
            password=md5(mobile[-6:]),
        )
        admin.save()

        user = M.User.create(
            mobile=mobile,
            username=name,
        )
        user.save()

    # apkversion
    apk = M.ApkVersion.create(version="1.0.0",
                              up_type=1,
                              url="/media/apk/smallbox.apk")
    apk.save()
Ejemplo n.º 9
0
def test_mapping_dist(metasync, opts):
    mapping = [("dropbox", 2), ("google", 15), ("box", 10), ("onedrive", 7), ("baidu", 2048)]
    mapping = map(lambda x:(util.md5(x[0])%10000,x[1]), mapping)
    print(mapping)
    hspace = (2+15+10+7+2048)*5
    objs = []
    with open("result/linux_objs.txt") as f:
        for line in f:
            sp = line.strip().split("\t")
            hv = sp[0] 
            size = int(sp[1])
            objs.append( (hv, size) )

    for replication in range(1, 4):
        detmap = DetMap2(mapping, hspace, replication)
        sizes = {}
        counts = {}
        for srv, sz in mapping:
            sizes[srv] = 0
            counts[srv] = 0

        for obj in objs:
            hv = obj[0]
            size = obj[1]
            lst = detmap.get_mapping(hv)
            for srv in lst:
                counts[srv] += 1
                sizes[srv] += size
        print replication, 
        for srv, sz in mapping:
            print "%d/%d" % (counts[srv],sizes[srv]),

        print
Ejemplo n.º 10
0
    def make_post_params(self, offset=0):
        #限制时间范围
        post_params = {
            'sdate': self.sdate_str,
            'edate': self.edate_str,
            '_t': int(time.time())
        }

        #限制统计类型
        if self.pm.statistics:
            post_params['type'] = ','.join(self.pm.statistics)
        if self.pm.is_create_time:

            post_params['is_create_time'] = 1

        #限制条目数
        post_params['offset'] = offset
        post_params['limit'] = self.LIMIT_NUM

        #设置签名
        post_params_str = urllib.urlencode(post_params)
        app_key = self.platform_model.app_key
        sign = md5('%s%s' % (post_params_str, app_key))
        post_params_str = '%s&sign=%s' % (post_params_str, sign)
        return post_params_str
Ejemplo n.º 11
0
    def __init__(self, url, save_path=None, status_func=None, done_func=None,
                 debug=False):
        self.url = url
        self.debug = debug

        if save_path is None:
            save_path = url.split('/')[-1]

        if save_path[0] != '/':
            save_path = path.join(os.getcwd(), save_path)
        self.save_path = save_path

        self.hash = util.md5(url)
        self.temp_dir = path.join(os.getcwd(), 'tmp')
        self.data_path = path.join(self.temp_dir, self.hash)

        self.meta = {}
        self.task_queue = multiprocessing.Queue()
        self.process_num = 10

        if hasattr(status_func, '__call__'):
            status_func("status")

        if hasattr(done_func, '__call__'):
            done_func('done')

        self.start_time = time.time()
Ejemplo n.º 12
0
def create_sign(data, extra_code):
    #print("-----------")
    #print(data)
    dict_keys = data.keys()
    #print(dict_keys)
    keys = list(dict_keys)
    keys.sort()
    #jjprint(keys)
    #print("-----------")
    #print [key for key in keys]
    #print [data[key] for key in keys]
    message = ""
    for k in keys:
        value = data[k]
        if k != "sign":
            if k != "param":
                #print(value)
                message = message + value
            else:
                tmp = json.dumps(value,separators=(',',':'))
                message = message + tmp
        
    message = message + extra_code
    print("message=%s" %message)
    sign = md5(message.encode("utf-8"))
    #print sign
    return sign
Ejemplo n.º 13
0
    def __init__(self,
                 url,
                 save_path=None,
                 status_func=None,
                 done_func=None,
                 process_num=10,
                 debug=False):
        self.url = url
        self.debug = debug

        if save_path is None:
            save_path = url.split('/')[-1]

        if save_path[0] != '/':
            save_path = os.path.join(os.getcwd(), save_path)
        self.save_path = save_path

        self.hash = util.md5(url)
        self.temp_dir = os.path.join(os.getcwd(), 'tmp')
        self.data_path = os.path.join(self.temp_dir, self.hash)

        self.meta = {}
        self.task_queue = multiprocessing.Queue()
        self.process_num = 10

        if hasattr(status_func, '__call__'):
            status_func("status")

        if hasattr(done_func, '__call__'):
            done_func('done')

        self.start_time = time.time()
Ejemplo n.º 14
0
 def succeed(self, url, detail_urls, batch_id):
     crawl_time = time.time()
     self.logger.info("parse %s successful" % url)
     sql_url = "UPDATE list_url SET crawl_time=FROM_UNIXTIME(%d), crawl_status=0, status=3, status_time=now() " \
             "WHERE url_md5='%s' AND batch_id=%d AND UNIX_TIMESTAMP(crawl_time) <= %d" % \
             (crawl_time, util.md5(url), batch_id, crawl_time)
     sql_client = "UPDATE client_list SET crawl_time=FROM_UNIXTIME(%d), crawl_num=crawl_num+1 "\
             "WHERE id=%d" % (crawl_time, self.id)
     sql_task_batch = "UPDATE task_batch SET last_crawl_time=FROM_UNIXTIME(%d), crawl_num=crawl_num+1, "\
             "parse_url_num=parse_url_num+%d WHERE task_type='%s' and batch_id=%d" % \
             (crawl_time, len(detail_urls), self.task_type, batch_id)
     self.db_operator(sql_url)
     self.db_operator(sql_client)
     self.db_operator(sql_task_batch)
     table_name = "detail_urls_%s" % time.strftime(
         "%Y%m%d", time.localtime(crawl_time))
     sql_create = """CREATE table IF NOT EXISTS %s (
     `id` int(11) NOT NULL auto_increment,
     `url` text NOT NULL,
     `list_url` text NOT NULL,
     `task_type` varchar(20) NOT NULL,
     `client_id` int(11) NOT NULL,
     `batch_id` int(11) NOT NULL default 1,
     `update_time` timestamp NOT NULL default CURRENT_TIMESTAMP,
     PRIMARY KEY  (`id`)
     ) ENGINE=MyISAM DEFAULT CHARSET=utf8""" % table_name
     self.db_operator(sql_create)
     sql_insert = "INSERT INTO %s(url, list_url, task_type, client_id, batch_id) VALUES" % table_name
     values = ",".join(["('%s', '%s', '%s', %d, %d)" % \
             (detail_url, url, self.task_type, self.id, batch_id) for detail_url in detail_urls])
     sql_insert += values
     self.db_operator(sql_insert)
Ejemplo n.º 15
0
def verify_file():
    filename = request.args.get('filename')
    checksum = request.args.get('checksum')
    filename = secure_filename(filename)
    basename = os.path.splitext(filename)[0]
    stagingdir = os.path.join(cfg.STAGING_FOLDER, basename)
    stagingpath = os.path.join(stagingdir, filename)

    if not os.path.exists(stagingpath):
        log.error('File %s does not exist', stagingpath)
        raise Error(message=('File %s does not exist' % stagingpath),
                    status_code=404)

    calculated_checksum = util.md5(stagingpath)

    valid = calculated_checksum == checksum
    if valid:
        log.info('File %s successfully verified', filename)
        return util.ret_ok()
    else:
        log.error('File %s: hash mismatch. Given: %s, calculated: %s',
                  filename, checksum, calculated_checksum)
        raise Error(message=('File hash mismatch. Given: %s, calculated: %s' %
                             (checksum, calculated_checksum)),
                    status_code=400)
Ejemplo n.º 16
0
def test_mapping_dist(metasync, opts):
    mapping = [("dropbox", 2), ("google", 15), ("box", 10), ("onedrive", 7),
               ("baidu", 2048)]
    mapping = map(lambda x: (util.md5(x[0]) % 10000, x[1]), mapping)
    print(mapping)
    hspace = (2 + 15 + 10 + 7 + 2048) * 5
    objs = []
    with open("result/linux_objs.txt") as f:
        for line in f:
            sp = line.strip().split("\t")
            hv = sp[0]
            size = int(sp[1])
            objs.append((hv, size))

    for replication in range(1, 4):
        detmap = DetMap2(mapping, hspace, replication)
        sizes = {}
        counts = {}
        for srv, sz in mapping:
            sizes[srv] = 0
            counts[srv] = 0

        for obj in objs:
            hv = obj[0]
            size = obj[1]
            lst = detmap.get_mapping(hv)
            for srv in lst:
                counts[srv] += 1
                sizes[srv] += size
        print replication,
        for srv, sz in mapping:
            print "%d/%d" % (counts[srv], sizes[srv]),

        print
Ejemplo n.º 17
0
	def get_posts(self,delay=DELAY,count=-1,save=False,path=None,tname=None):
		_count = 0
		results = []
		_check = count if count > 0 else self.posts_count
		top_posts_card = self.info['edge_owner_to_timeline_media']
		top_posts = top_posts_card['edges']
		end_cursor = top_posts_card['page_info']['end_cursor']
		posts_query_id = self.queryIds[2]
		headers = COMMON_HEADERS
		headers['x-ig-app-id']=self.app_id
		for i in top_posts:
			if save:
				self.db.save(i,tname=tname)
			_count += 1
			results.append(i)
			if (_count >= count or _count >= self.posts_count) and (count > 0):
				logger.info(f'[Done]The length of crawled data of user "{self.name}"'
					f':{len(results)}.[Total({self.posts_count})]')
				return results
		logger.info(f'Total posts of user "{self.name}":{self.posts_count}.')
		while 1:
			if not end_cursor:
				logger.info(f'[Done]The length of crawled data of user "{self.name}"'
					f':{len(results)}.[Total({self.posts_count})]')
				break
			params = {}
			params['query_hash']=posts_query_id
			params['variables']=r'{"id":"'+self.Id+'","first":"'+\
			str(USER_POSTS_MAX)+'","after":"'+end_cursor+'"}'
			md5ed = md5(self.rhx_gis + ":" + params['variables'])
			headers['x-instagram-gis']=md5ed
			response = send_request(API_USER_POSTS,
				params=params,
				headers=headers,
				delay=delay,
				json=True,
				proxy=PROXY_GLOBAL)
			json_data = response.json()
			data = json_data['data']['user']\
			['edge_owner_to_timeline_media']['edges']
			page_info = json_data['data']['user']\
			['edge_owner_to_timeline_media']['page_info']
			for i in data:
				if save:
					self.db.save(i,tname=tname)
				results.append(i)
				_count += 1
				if (_count >= count or _count >= self.posts_count) and (count > 0):
					logger.info(f'[Done]The length of crawled data of user "{self.name}"'
						f':{len(results)}.[Total({self.posts_count})]')
					return results
			logger.info(f'Current amount of posts of user "{self.name}"'
				f':{len(results)}.[{round(len(results)/_check,4)*100 if _check else 0}%]')
			end_cursor = page_info['end_cursor']
			if not page_info['has_next_page']:
				logger.info(f'[Done]The length of crawled data of user "{self.name}"'
					f':{len(results)}.[Total({self.posts_count})]')
				break
		return results
Ejemplo n.º 18
0
def post_headers(agentConfig, payload):
    return {
        'User-Agent': 'Datadog Agent/%s' % agentConfig['version'],
        'Content-Type': 'application/json',
        'Content-Encoding': 'deflate',
        'Accept': 'text/html, */*',
        'Content-MD5': md5(payload).hexdigest()
    }
Ejemplo n.º 19
0
def old_login(username, md5_password):

    hash_password = rsa_encode(md5_password)

    _chars = "0123456789ABCDEF"
    peer_id = ''.join(random.sample(_chars, 16))

    device_id = md5("%s23333" % md5_password) # just generate a 32bit string

    appName = 'com.xunlei.redcrystalandroid'
    key = 'C2049664-1E4A-4E1C-A475-977F0E207C9C'
    md5_key = md5(key)

    device_sign = 'div100.%s%s' % (device_id, md5(sha1("%s%s%s%s" % (device_id, appName, 61, md5_key))))

    payload = json.dumps({
            "protocolVersion": PROTOCOL_VERSION,
            "sequenceNo": 1000001,
            "platformVersion": 1,
            "sdkVersion": 177588,
            "peerID": peer_id,
            "businessType": 61,
            "clientVersion": APP_VERSION,
            "devicesign": device_sign,
            "isCompressed": 0,
            "cmdID": 1,
            "userName": username,
            "passWord": hash_password,
            "loginType": 0,
            "sessionID": "",
            "verifyKey": "",
            "verifyCode": "",
            "appName": "ANDROID-com.xunlei.redcrystalandroid",
            "rsaKey": {
                "e": "%06X" % rsa_pubexp,
                "n": long2hex(rsa_mod)
            },
            "extensionList": ""
    })

    headers = {'User-Agent': "Mozilla/5.0 (iPhone; CPU iPhone OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Mobile/9B176 MicroMessenger/4.3.2"}
    r = requests.post("https://login.mobile.reg2t.sandai.net/", data=payload, headers=headers, verify=False)

    login_status = json.loads(r.text)

    return login_status
Ejemplo n.º 20
0
def post_headers(agentConfig, payload):
    return {
        'User-Agent': 'Datadog Agent/%s' % agentConfig['version'],
        'Content-Type': 'application/json',
        'Content-Encoding': 'deflate',
        'Accept': 'text/html, */*',
        'Content-MD5': md5(payload).hexdigest()
    }
Ejemplo n.º 21
0
	def get_channel_posts(self,delay=DELAY,count=-1,save=False,path=None,tname=None):
		_count = 0
		results = []
		_check = count if count > 0 else self.channel_posts_count
		top_posts_card = self.info['edge_felix_video_timeline']
		top_posts = top_posts_card['edges']
		end_cursor = top_posts_card['page_info']['end_cursor']
		headers = COMMON_HEADERS
		headers['x-ig-app-id'] = self.app_id
		for i in top_posts:
			if save:
				self.db.save(i,tname=tname)
			_count += 1
			results.append(i)
			if (_count >= count or _count >= self.channel_posts_count) and (count > 0):
				logger.info(f'[Done]The amount of crawled channel posts data of user "{self.name}":{len(results)}.'
					f'[Total({self.channel_posts_count})]')
				return results
		logger.info(f'Total channel posts of user "{self.name}":{self.channel_posts_count}.')
		while 1:
			if not end_cursor:
				logger.info(f'[Done]The amount of crawled channel posts data of user "{self.name}":{len(results)}.'
					f'[Total({self.channel_posts_count})]')
				break
			params = copy.deepcopy(CHANNEL_PARAMS)
			params['variables'] = params['variables'].replace('%',self.Id).replace('$',end_cursor)
			params['query_hash'] = self.channel_hash
			md5ed = md5(self.rhx_gis + ":" + params['variables'])
			headers['x-instagram-gis']=md5ed
			response = send_request(API_USER_POSTS,
				session=self.instagram.session,
				params=params,
				headers=headers,
				delay=delay,
				json=True,
				proxy=PROXY_GLOBAL)
			json_data = response.json()
			posts = json_data['data']['user']['edge_felix_video_timeline']['edges']
			page_info = json_data['data']['user']['edge_felix_video_timeline']['page_info']
			has_next_page = page_info['has_next_page']
			end_cursor = page_info['end_cursor']
			for i in posts:
				if save:
					self.db.save(i,tname=tname)
				results.append(i)
				_count += 1
				if (_count >= count or _count >= self.channel_posts_count) and (count > 0):
					logger.info(f'[Done]The amount of crawled channel posts data of user "{self.name}"'
						f':{len(results)}.[Total({self.channel_posts_count})]')
					return results
			logger.info(f'Current amount of crawled channel posts data of user "{self.name}"'
				f':{len(results)}.[{round(len(results)/_check,4)*100 if _check else 0}%]')
			if not has_next_page:
				logger.info(f'[Done]The amount of crawled channel posts data of user "{self.name}"'
					f':{len(results)}.[Total({self.channel_posts_count})]')
				break
		return results
Ejemplo n.º 22
0
    def testSpeed(self):
        # Pretend to be gmetad and serve a large piece of content
        server = subprocess.Popen("nc -l 8651 < %s" % TEST_FN, shell=True)
        # Wait for 1 second
        time.sleep(1)

        pfile = tempfile.NamedTemporaryFile()
        g = Ganglia(logging.getLogger(__file__))
        # Running the profiler
        # profile.runctx("g.check({'ganglia_host': 'localhost', 'ganglia_port': 8651})", {}, {"g": g}, pfile.name)
        # p = pstats.Stats(pfile.name)
        # p.sort_stats('time').print_stats()
        self.assertEquals(
            md5(g.check({
                'ganglia_host': 'localhost',
                'ganglia_port': 8651
            })).hexdigest(),
            md5(open(TEST_FN).read()).hexdigest())
Ejemplo n.º 23
0
    def updateFiles(self, destination, filegroup):
        """
        Updates the files in a given file group, in the destination subdirectory of the Forged Alliance path.
        If existing=True, the existing contents of the directory will be added to the current self.filesToUpdate
        list. 
        """
        QtWidgets.QApplication.processEvents()

        self.progress.setLabelText("Updating files: " + filegroup)
        self.destination = destination

        self.connection.writeToServer("GET_FILES_TO_UPDATE", filegroup)
        self.waitForFileList()

        # Ensure our list is unique
        self.filesToUpdate = list(set(self.filesToUpdate))

        targetdir = os.path.join(util.APPDATA_DIR, destination)
        if not os.path.exists(targetdir):
            os.makedirs(targetdir)

        for fileToUpdate in self.filesToUpdate:
            md5File = util.md5(
                os.path.join(util.APPDATA_DIR, destination, fileToUpdate))
            if md5File is None:
                if self.version:
                    if self.featured_mod == "faf" or self.featured_mod == "ladder1v1" or \
                                    filegroup == "FAF" or filegroup == "FAFGAMEDATA":
                        self.connection.writeToServer("REQUEST_VERSION",
                                                      destination,
                                                      fileToUpdate,
                                                      str(self.version))
                    else:
                        self.connection.writeToServer(
                            "REQUEST_MOD_VERSION", destination, fileToUpdate,
                            json.dumps(self.modversions))
                else:

                    self.connection.writeToServer("REQUEST_PATH", destination,
                                                  fileToUpdate)
            else:
                if self.version:
                    if self.featured_mod == "faf" or self.featured_mod == "ladder1v1" or \
                                    filegroup == "FAF" or filegroup == "FAFGAMEDATA":
                        self.connection.writeToServer("PATCH_TO", destination,
                                                      fileToUpdate, md5File,
                                                      str(self.version))
                    else:

                        self.connection.writeToServer(
                            "MOD_PATCH_TO", destination, fileToUpdate, md5File,
                            json.dumps(self.modversions))
                else:
                    self.connection.writeToServer("UPDATE", destination,
                                                  fileToUpdate, md5File)

        self.waitUntilFilesAreUpdated()
Ejemplo n.º 24
0
def old_login(username, md5_password):
    from api import agent_header
    exponent = int("010001", 16)
    modulus = int(
        "AC69F5CCC8BDE47CD3D371603748378C9CFAD2938A6B021E0E191013975AD683F5CBF9ADE8BD7D46B4D2EC2D78A"
        "F146F1DD2D50DC51446BB8880B8CE88D476694DFC60594393BEEFAA16F5DBCEBE22F89D640F5336E42F587DC4AF"
        "EDEFEAC36CF007009CCCE5C1ACB4FF06FBA69802A8085C2C54BADD0597FC83E6870F1E36FD",
        16)

    param = '{"cmdID":1,"isCompressed":0,"rsaKey":{"n":"AC69F5CCC8BDE47CD3D371603748378C9CFAD2938A6B0' \
            '21E0E191013975AD683F5CBF9ADE8BD7D46B4D2EC2D78AF146F1DD2D50DC51446BB8880B8CE88D476694DFC60594393BEEFAA16F' \
            '5DBCEBE22F89D640F5336E42F587DC4AFEDEFEAC36CF007009CCCE5C1ACB4FF06FBA69802A8085C2C54BADD0597FC83E6870F1E3' \
            '6FD","e":"010001"},"businessType":%s,"passWord":"******","loginType":0,"sdkVersion":177588,' \
            '"appName":"ANDROID-com.xunlei.redcrystalandroid","platformVersion":1,"devicesign":"%s",' \
            '"sessionID":"","protocolVersion":%s,"userName":"******","extensionList":"","sequenceNo":%s,' \
            '"peerID":"","clientVersion":"1.0.0"}'

    _chars = "0123456789ABCDEF"

    deviceid = username
    device_id = md5(deviceid)

    appName = 'com.xunlei.redcrystalandroid'
    businessType = '61'
    key = 'C2049664-1E4A-4E1C-A475-977F0E207C9C'
    key_md5 = md5(key)

    device_sign = "div100.%s%s" % (
        device_id,
        md5(sha1("%s%s%s%s" % (device_id, appName, businessType, key_md5))))

    hash_password = hex(pow_mod(StrToInt(md5_password), exponent,
                                modulus))[2:].upper().zfill(256)

    params = param % (61, hash_password, device_sign, 108, username, 1000006)

    r = requests.post("https://login.mobile.reg2t.sandai.net/",
                      data=params,
                      headers=agent_header,
                      verify=False)
    login_status = json.loads(r.text)

    return login_status
Ejemplo n.º 25
0
 def legalFAVersion(self):
     #Now we check the FA version
     luascd = os.path.join(self.path, 'gamedata', 'lua.scd')
             
     if not os.path.isfile(luascd):
         return False
         
     md5LUA = util.md5(luascd)
     log("lua.scd digest is %s" % md5LUA)
     return md5LUA in ["4af45c46b0abb2805bea192e6e2517d4","5cdd99bddafa38f0873bd11dd353055a","ad999839d32f4784e331e5e8be1d32a2"]  
Ejemplo n.º 26
0
def account_add():
    user = session.get('user_info')

    account_name = request.values.get('xl_username')
    password = request.values.get('xl_password')

    md5_password = md5(password)

    user_key = '%s:%s' % ('user', user.get('username'))
    user_info = json.loads(r_session.get(user_key).decode('utf-8'))

    if user_info.get('max_account_no') is None:
        user_info['max_account_no'] = 1

    accounts_key = 'accounts:%s' % user.get('username')

    account_no = r_session.scard(accounts_key)
    if account_no is not None:
        if account_no >= user_info.get('max_account_no'):
            session['error_message'] = '你的账号限制%d个账户。' % account_no
            return redirect(url_for('accounts'))

    login_result = login(account_name, md5_password,
                         app.config.get('ENCRYPT_PWD_URL'))
    if login_result.get('errorCode') != 0:
        error_message = login_result.get('errorDesc')
        session['error_message'] = '登陆失败,错误信息:%s。' % error_message
        return redirect(url_for('accounts'))

    xl_session_id = login_result.get('sessionID')
    xl_nick_name = login_result.get('nickName')
    xl_user_name = login_result.get('userName')
    xl_user_id = str(login_result.get('userID'))
    xl_user_new_no = str(login_result.get('userNewNo'))
    xl_account_name = account_name
    xl_password = md5_password

    r_session.sadd(accounts_key, xl_user_id)

    account_key = 'account:%s:%s' % (user.get('username'), xl_user_id)
    xl_account_data = dict(
        session_id=xl_session_id,
        nick_name=xl_nick_name,
        username=xl_user_name,
        user_id=xl_user_id,
        user_new_no=xl_user_new_no,
        account_name=xl_account_name,
        password=xl_password,
        active=True,
        status='OK',
        createdtime=datetime.now().strftime('%Y-%m-%d %H:%M'))
    r_session.set(account_key, json.dumps(xl_account_data))

    return redirect(url_for('accounts'))
Ejemplo n.º 27
0
    def get_best_results(self, n_top_rows):

        by = self.get_result_column_index()
        df = pd.DataFrame(self.data)

        df = df.sort_values(by=by, ascending=False)
        df['md5'] = df.apply(lambda x: md5(str(tuple(x))), axis=1)
        #temporaly display all columns
        with pd.option_context('display.max_rows', None):
            print(df.head(n_top_rows))
        return df.head(n_top_rows).values
Ejemplo n.º 28
0
def account_add():
    session['action']='one'

    account_name = request.values.get('xl_username')
    password = request.values.get('xl_password')
    md5_password = md5(password)

    user = session.get('user_info')

    accounts_key = 'accounts:%s' % user.get('username')

    user_key = '%s:%s' % ('user', user.get('username'))
    user_info = json.loads(r_session.get(user_key).decode('utf-8'))
    account_no = r_session.scard(accounts_key) + 1
    if user_info.get('account_limit') is not None and user_info['account_limit'] < account_no:
        session['error_message']='账户数量已达上限,无法完成添加'
        return redirect(url_for('accounts'))
    if account_no is not None:
        if account_no >= user_info.get('total_account_point'):
            session['error_message']='账户余额不足,无法完成添加'
            return redirect(url_for('accounts'))
        elif account_no >= user_info.get('max_account_no'):
            user_info['max_account_no'] = account_no
            days=int(user_info.get('total_account_point') / user_info.get('max_account_no'))
            if days<36500:
                user_info['expire_date'] = (datetime.now() + timedelta(days=days)).strftime('%Y-%m-%d')
            else:
                user_info['expire_date'] = (datetime.now() + timedelta(days=36500)).strftime('%Y-%m-%d')
            r_session.set(user_key, json.dumps(user_info))

    login_result = login(account_name, md5_password, app.config.get('ENCRYPT_PWD_URL'))
    if login_result.get('errorCode') != 0:
        error_message = login_result.get('errorDesc')
        session['error_message'] = '登陆失败,错误信息:%s。' % error_message
        return redirect(url_for('accounts'))

    xl_session_id = login_result.get('sessionID')
    xl_nick_name = login_result.get('nickName')
    xl_user_name = login_result.get('userName')
    xl_user_id = str(login_result.get('userID'))
    xl_user_new_no = str(login_result.get('userNewNo'))
    xl_account_name = account_name
    xl_password = md5_password

    r_session.sadd(accounts_key, xl_user_id)

    account_key = 'account:%s:%s' % (user.get('username'), xl_user_id)
    xl_account_data = dict(session_id=xl_session_id, nick_name=xl_nick_name, username=xl_user_name,
                           user_id=xl_user_id, user_new_no=xl_user_new_no, account_name=xl_account_name,
                           password=xl_password, active=True, status='OK',
                           createdtime=datetime.now().strftime('%Y-%m-%d %H:%M'))
    r_session.set(account_key, json.dumps(xl_account_data))

    return redirect(url_for('accounts'))
Ejemplo n.º 29
0
 def default_template_file_action(self,
                                  dir_name,
                                  file_name,
                                  dest_rel_path=None,
                                  dest_name=None):
     template_full = dir_name + '/' + file_name
     Config.log("default_template_file_action '%s'" % template_full,
                tag='DEFAULT_TEMPLATE_FILE_ACTION')
     if dest_name:
         rel_path = dest_rel_path
         dest_path = u.pathify(self.output_root, dest_rel_path)
     else:
         rel_path = u.make_rel_path(self.site_root, dir_name)
         dest_path = u.pathify(self.output_root, rel_path)
         dest_name = file_name
     u.ensure_path(dest_path)
     dest_full = u.pathify(dest_path, dest_name)
     info = {
         'name': dest_name,
         'path': dest_path,
         'rel_path': rel_path,
         'full': dest_full,
         'key': u.make_key(rel_path, dest_name)
     }
     if self.config.is_template_type(file_name):
         template = open(template_full).read()
         output = u.debracket(template, self.interpret)
         if not self.config.is_special_file(info['key']):
             open(dest_full, 'w').write(output)
             local = u.local_metadata(dest_path, dest_name)
             info['size'] = local['size']
             info['modified'] = local['modified']
             info['md5'] = u.md5(dest_full)
             self.track_file(info)
     else:
         shutil.copyfile(template_full, dest_full)
         local = u.local_metadata(dest_path, dest_name)
         info['size'] = local['size']
         info['modified'] = local['modified']
         info['md5'] = u.md5(dest_full)
         self.track_file(info)
Ejemplo n.º 30
0
 def get_by_client(self, task_type, resp):
     if not self.conn: return 0
     if not resp: return 0
     item = json.loads(resp)
     url = item['url'].encode("utf-8", "ignore")
     _id = util.md5(url)[:24]        
     try:
         self.conn[task_type].update({'_id': ObjectId(_id)},
                 {'$set': {'Status': 2, 'StatusTime': datetime.datetime.now()}},
                 multi = True)
     except Exception, e:
         print 'error:', type(url), _id
Ejemplo n.º 31
0
 def _load(self, config):
     nspace = []
     for n, size in config:
         for s in range(size):
             nspace.append((n,s))
     assert len(nspace) < self.hspace
     self.detmap = [None] * self.hspace
     for i in range(self.hspace):
         group = []
         for n in nspace:
             order = int(util.md5("%s.%s" % (i, n)))
             group.append((n[0], order))
         self.detmap[i] = sorted(group, key=lambda e:e[1])
Ejemplo n.º 32
0
    def testSpeed(self):
        # Pretend to be gmetad and serve a large piece of content
        server = subprocess.Popen("nc -l 8651 < %s" % TEST_FN, shell=True)
        # Wait for 1 second
        time.sleep(1)

        pfile = tempfile.NamedTemporaryFile()
        g = Ganglia(logging.getLogger(__file__))
        # Running the profiler
        # profile.runctx("g.check({'ganglia_host': 'localhost', 'ganglia_port': 8651})", {}, {"g": g}, pfile.name)
        # p = pstats.Stats(pfile.name)
        # p.sort_stats('time').print_stats()
        self.assertEquals(md5(g.check({'ganglia_host': 'localhost', 'ganglia_port': 8651})).hexdigest(), md5(open(TEST_FN).read()).hexdigest())
Ejemplo n.º 33
0
 def _load(self, config):
     nspace = []
     for n, size in config:
         for s in range(size):
             nspace.append((n, s))
     assert len(nspace) < self.hspace
     self.detmap = [None] * self.hspace
     for i in range(self.hspace):
         group = []
         for n in nspace:
             order = int(util.md5("%s.%s" % (i, n)))
             group.append((n[0], order))
         self.detmap[i] = sorted(group, key=lambda e: e[1])
Ejemplo n.º 34
0
 def cache_view(self,request, view_func, view_args, view_kwargs):
     '''页面级别的缓存
     '''
     cache_method_list = ['']
     _mc = MemcachedUtil()
     params = str(request.REQUEST)
     view_func_name = view_func.__name__
     key = md5('%s_%s_%s' % (request.admin.id,view_func.__name__,params))
     response = _mc.get(key)
     if not response:
         response = view_func(request,*view_args,**view_kwargs)
         _mc.set(key, response, 1800)
     return response
Ejemplo n.º 35
0
    def geo_search(self, call='query', query=None):

        if not query:
            raise Exception("geo_search: query cannot be empty")

        now = datetime.datetime.now()

        day_output_folder = os.path.abspath(
            '%s/%s' % (self.output_folder, now.strftime('%Y%m%d')))

        if not os.path.exists(day_output_folder):
            os.makedirs(day_output_folder)

        filename = os.path.abspath(
            '%s/%s' % (day_output_folder, util.md5(query.encode('utf-8'))))

        retry_cnt = MAX_RETRY_CNT
        while retry_cnt > 0:
            try:

                result = None

                if ('query' == call):
                    result = self.search_geo(query=query)
                elif ('ip' == call):
                    result = self.search_geo(ip=query)
                else:
                    logger.error("call == ('query' or 'ip')")
                    return

                if (result):
                    with open(filename, 'a+', newline='',
                              encoding='utf-8') as f:
                        f.write('%s\n' % json.dumps(result))

                time.sleep(1)

                return False

            except twython.exceptions.TwythonRateLimitError:
                self.rate_limit_error_occured('geo', '/geo/search')
            except Exception as exc:
                time.sleep(10)
                logger.error("exception: %s; when fetching place: %s" %
                             (exc, query))
                retry_cnt -= 1
                if (retry_cnt == 0):
                    logger.warn("exceed max retry... return")
                    return

        return
Ejemplo n.º 36
0
    def check(self, instance):
        url = instance['url']
        default_timeout = self.init_config.get('default_timeout', 5)
        timeout = float(instance.get('timeout', default_timeout))

        aggregation_key = md5(url).hexdigest()

        try:
            h = Http(timeout=timeout)
            resp, content = h.request(url, "GET")

        except socket.timeout, e:
            self.timeout_event(url, timeout, aggregation_key)
            return
Ejemplo n.º 37
0
 def parse_failed(self, url, batch_id):
     self.logger.error("parse %s failed" % url)
     crawl_time = time.time()
     sql_url = "UPDATE list_url SET crawl_time=FROM_UNIXTIME(%d), crawl_status=2, status=3, status_time=now() " \
             "WHERE url_md5='%s' AND batch_id=%d, UNIX_TIMESTAMP(crawl_time) <= %d" % \
             (crawl_time, util.md5(url), batch_id, crawl_time)
     sql_client = "UPDATE client_list SET crawl_time=FROM_UNIXTIME(%d), crawl_num=crawl_num+1, " \
             "parse_failed_num=parse_failed_num+1 WHERE id=%d" % (crawl_time, self.id)
     sql_task_batch = "UPDATE task_batch SET last_crawl_time=FROM_UNIXTIME(%d), crawl_num=crawl_num+1, "\
             "parse_failed_num=parse_failed_num+1 "\
             "WHERE task_type='%s' and batch_id=%d" % (crawl_time, self.task_type, batch_id)
     self.db_operator(sql_url)
     self.db_operator(sql_client)
     self.db_operator(sql_task_batch)
Ejemplo n.º 38
0
    def check(self, instance):
        url             = instance['url']
        default_timeout = self.init_config.get('default_timeout', 5)
        timeout         = float(instance.get('timeout', default_timeout))

        aggregation_key = md5(url).hexdigest()

        try:
            h = Http(timeout=timeout)
            resp, content = h.request(url, "GET")

        except socket.timeout, e:
            self.timeout_event(url, timeout, aggregation_key)
            return
Ejemplo n.º 39
0
 def _upload(self, src_path, src_name, key, bucket=None, content_type=None, extra_args=None):
     full = u.pathify(src_path, src_name)
     if extra_args is None:
         extra_args = {
             'Metadata': {
                 'md5': u.md5(full)
             }
         }
     if not content_type:
         content_type = mimetypes.MimeTypes().guess_type(src_name)[0]
         extra_args['ContentType'] = content_type
     dest_full = self._file_dest_root + '/' + key
     dest_dir, dest_name = os.path.split(dest_full)
     u.ensure_path(dest_dir)
     shutil.copyfile(full, dest_full)
Ejemplo n.º 40
0
    def geo_search(self, call='query', query=None):

        if not query:
            raise Exception("geo_search: query cannot be empty")

        now=datetime.datetime.now()

        day_output_folder = os.path.abspath('%s/%s'%(self.output_folder, now.strftime('%Y%m%d')))

        if not os.path.exists(day_output_folder):
            os.makedirs(day_output_folder)

        filename = os.path.abspath('%s/%s'%(day_output_folder, util.md5(query.encode('utf-8'))))

        retry_cnt = MAX_RETRY_CNT
        while retry_cnt > 0:
            try:
                
                result = None

                if ('query' == call):
                    result = self.search_geo(query=query)
                elif ('ip' == call):
                    result = self.search_geo(ip=query)
                else:
                    logger.error("call == ('query' or 'ip')")
                    return

                if (result):
                    with open(filename, 'a+', newline='', encoding='utf-8') as f:
                        f.write('%s\n'%json.dumps(result))

                time.sleep(1)
                
                return False

            except twython.exceptions.TwythonRateLimitError:
                self.rate_limit_error_occured('geo', '/geo/search')
            except Exception as exc:
                time.sleep(10)
                logger.error("exception: %s; when fetching place: %s"%(exc, query))
                retry_cnt -= 1
                if (retry_cnt == 0):
                    logger.warn("exceed max retry... return")
                    return

        return
Ejemplo n.º 41
0
def account_add():
    # 获取表单提交的迅雷会员名和帐户密码
    account_name = request.values.get('xl_username')
    password = request.values.get('xl_password')
    md5_password = md5(password)

    user = session.get('user_info')

    accounts_key = 'accounts:%s' % user.get('username')

    if user.get('max_account_no') is None:
        user['max_account_no'] = 5

    account_no = r_session.scard(accounts_key)

    if account_no is not None:
        if account_no >= user.get('max_account_no'):
            session['error_message'] = '你的账号限制%d个账户。' % account_no
            return redirect(url_for('accounts'))

    login_result = login(account_name, md5_password,
                         app.config.get('ENCRYPT_PWD_URL'))
    if login_result.get('errorCode') != 0:
        error_message = login_result.get('errorDesc')
        session['error_message'] = '登陆失败,错误信息:%s。' % error_message
        return redirect(url_for('accounts'))

    xl_session_id = login_result.get('sessionID')
    xl_nick_name = login_result.get('nickName')
    xl_user_name = login_result.get('userName')
    xl_user_id = str(login_result.get('userID'))
    xl_user_new_no = str(login_result.get('userNewNo'))
    xl_account_name = account_name
    xl_password = md5_password

    r_session.sadd(accounts_key, xl_user_id)

    account_key = 'account:%s:%s' % (user.get('username'), xl_user_id)
    xl_account_data = dict(session_id=xl_session_id, nick_name=xl_nick_name, username=xl_user_name,
                           user_id=xl_user_id, user_new_no=xl_user_new_no, account_name=xl_account_name,
                           password=xl_password, active=True, status='OK',
                           createdtime=datetime.now().strftime('%Y-%m-%d %H:%M'))
    r_session.set(account_key, json.dumps(xl_account_data))

    return redirect(url_for('accounts'))
Ejemplo n.º 42
0
    def updateFiles(self, destination, filegroup):
        """
        Updates the files in a given file group, in the destination subdirectory of the Forged Alliance path.
        If existing=True, the existing contents of the directory will be added to the current self.filesToUpdate
        list. 
        """
        QtGui.QApplication.processEvents()

        self.progress.setLabelText("Updating files: " + filegroup)
        self.destination = destination

        self.writeToServer("GET_FILES_TO_UPDATE", filegroup)
        self.waitForFileList()

        #Ensure our list is unique
        self.filesToUpdate = list(set(self.filesToUpdate))

        targetdir = os.path.join(util.APPDATA_DIR, destination)
        if not os.path.exists(targetdir):
            os.makedirs(targetdir)

        for fileToUpdate in self.filesToUpdate:
            md5File = util.md5(os.path.join(util.APPDATA_DIR, destination, fileToUpdate))
            if md5File == None:
                if self.version:
                    if self.featured_mod == "faf" or self.featured_mod == "ladder1v1" or filegroup == "FAF" or filegroup == "FAFGAMEDATA":
                        self.writeToServer("REQUEST_VERSION", destination, fileToUpdate, str(self.version))
                    else:
                        self.writeToServer("REQUEST_MOD_VERSION", destination, fileToUpdate,
                                           json.dumps(self.modversions))
                else:

                    self.writeToServer("REQUEST_PATH", destination, fileToUpdate)
            else:
                if self.version:
                    if self.featured_mod == "faf" or self.featured_mod == "ladder1v1" or filegroup == "FAF" or filegroup == "FAFGAMEDATA":
                        self.writeToServer("PATCH_TO", destination, fileToUpdate, md5File, str(self.version))
                    else:

                        self.writeToServer("MOD_PATCH_TO", destination, fileToUpdate, md5File,
                                           json.dumps(self.modversions))
                else:
                    self.writeToServer("UPDATE", destination, fileToUpdate, md5File)

        self.waitUntilFilesAreUpdated()
Ejemplo n.º 43
0
 def get(self):
   form = forms.BookmarkletForm(self)
   if form.validate():
     if not form.title.data:
       form.title.data = form.url.data
     url_digest = util.md5(form.url.data)
     bookmark = self.db.bookmarks.find_one({
         'user': self.current_user._id,
         'url_digest': url_digest})
     if bookmark is None:
       bookmark = {'user': self.current_user._id,
                   'modified': datetime.datetime.now()}
     bookmark = tornado.web._O(bookmark)
     form.populate_obj(bookmark)
     self.db.bookmarks.save(bookmark)
     self.write('oldu')
   else:
     self.write('%s' % form.errors)
Ejemplo n.º 44
0
 def download_textures(self):
     self.progress.show()
     self.progress.setLabelText("Downloading resources ...")
     
     textInCache = []
     
     for tex in self.texturelist : 
         if os.path.exists(self.get_texture_name(tex)) :
             if util.md5(self.get_texture_name(tex)) == self.texturelist[tex] :
                 #logger.debug(tex + ".png in cache.")
                 textInCache.append(tex)
                 continue
         #logger.debug("Downloading " + tex + ".png")
         self.downloader.get(QNetworkRequest(QtCore.QUrl(TEXTURE_SERVER + tex + ".png")))    
     
     for tex in textInCache :
         del self.texturelist[tex]
     
     if len(self.texturelist) == 0 :
         self.progress.close()
         self.setup()
Ejemplo n.º 45
0
 def sid(self):
   return util.md5("baidu") % 10000
Ejemplo n.º 46
0
 def sid(self):
   return util.md5("google") % 10000
Ejemplo n.º 47
0
    def search_by_query(self, query, since_id = 0, geocode=None, lang=None, output_filename = None):

        if not query:
            raise Exception("search: query cannot be None")

        now=datetime.datetime.now()
        #logger.info("query: %s; since_id: %d"%(query, since_id))
        place = None
        geo = None

        if (geocode):
            place, geo = geocode

            day_output_folder = os.path.abspath('%s/%s/%s'%(self.output_folder, now.strftime('%Y%m%d'), place))
        else:
            day_output_folder = os.path.abspath('%s/%s'%(self.output_folder, now.strftime('%Y%m%d')))

        if not os.path.exists(day_output_folder):
            os.makedirs(day_output_folder)

        filename = os.path.abspath('%s/%s'%(day_output_folder, util.md5(query.encode('utf-8'))))

        prev_max_id = -1

        current_max_id = 0
        cnt = 0
        current_since_id = since_id

        retry_cnt = MAX_RETRY_CNT
        #result_tweets = []
        while current_max_id != prev_max_id and retry_cnt > 0:
            try:
                if current_max_id > 0:
                    tweets = self.search(q=query, geocode=geo, since_id=since_id, lang=lang, tweet_mode='extended', max_id=current_max_id-1, result_type='recent', count=100)
                else:
                    tweets = self.search(q=query, geocode=geo, since_id=since_id, lang=lang, tweet_mode='extended', result_type='recent', count=100)


                prev_max_id = current_max_id # if no new tweets are found, the prev_max_id will be the same as current_max_id

                with open(filename, 'a+', newline='', encoding='utf-8') as f:
                    for tweet in tweets['statuses']:
                        f.write('%s\n'%json.dumps(tweet))
                        if current_max_id == 0 or current_max_id > int(tweet['id']):
                            current_max_id = int(tweet['id'])
                        if current_since_id == 0 or current_since_id < int(tweet['id']):
                            current_since_id = int(tweet['id'])

                #no new tweets found
                if (prev_max_id == current_max_id):
                    break

                #result_tweets.extend(tweets['statuses'])

                cnt += len(tweets['statuses'])

                # if (cnt % 1000 == 0):
                #     logger.info("[%d] tweets... "%cnt)

                #logger.info(cnt)

                #logger.debug('%d > %d ? %s'%(prev_max_id, current_max_id, bool(prev_max_id > current_max_id)))

                time.sleep(1)

            except twython.exceptions.TwythonRateLimitError:
                self.rate_limit_error_occured('search', '/search/tweets')
            except Exception as exc:
                time.sleep(10)
                logger.error("exception: %s"%exc)
                retry_cnt -= 1
                if (retry_cnt == 0):
                    logger.warn("exceed max retry... return")
                    return since_id
                    #raise MaxRetryReached("max retry reached due to %s"%(exc))

        logger.info("[%s]; since_id: [%d]; total tweets: %d "%(query, since_id, cnt))
        return current_since_id
Ejemplo n.º 48
0
def collect_tweets_by_search_terms(search_configs_filepath, output_folder, config):
    
    apikeys = list(config['apikeys'].values()).pop()

    search_configs = {}
    with open(os.path.abspath(search_configs_filepath), 'r') as search_configs_rf:
        search_configs = json.load(search_configs_rf)

    for search_config_id in itertools.cycle(search_configs):
       
        search_config = search_configs[search_config_id]

        search_terms = [term.lower() for term in search_config['terms']]
        querystring = '%s'%(' OR '.join('(' + term + ')' for term in search_terms))
        since_id = search_config['since_id'] if 'since_id' in search_config else 0
        geocode = tuple(search_config['geocode']) if ('geocode' in search_config and search_config['geocode']) else None

        logger.info('REQUEST -> (md5(querystring): [%s]; since_id: [%d]; geocode: [%s])'%(util.md5(querystring.encode('utf-8')), since_id, geocode))


        try:
            twitterCralwer = TwitterCrawler(apikeys=apikeys, client_args=CLIENT_ARGS, output_folder = output_folder)
            since_id = twitterCralwer.search_by_query(querystring, geocode = geocode, since_id = since_id)
        except Exception as exc:
            logger.error(exc)
            logger.error(util.full_stack())
            pass

        search_config['since_id'] = since_id
        search_config['querystring'] = querystring
        search_config['geocode'] = geocode

        search_configs[search_config_id] = search_config

        flash_cmd_config(search_configs, search_configs_filepath, output_folder)

        logger.info('COMPLETED -> (md5(querystring): [%s]; since_id: [%d]; geocode: [%s])'%(util.md5(querystring.encode('utf-8')), since_id, geocode))
        logger.info('PAUSE %ds to CONTINUE...'%WAIT_TIME)
        time.sleep(WAIT_TIME)
Ejemplo n.º 49
0
 def sid(self):
   return util.md5("onedrive") % 10000
Ejemplo n.º 50
0
 def __init__(self, root):
     self.root = root
     util.mkdirs(root)
     self._sid  = util.md5(self.root) % 10000
Ejemplo n.º 51
0
 def sid(self):
   return util.md5("dropbox") % 10000