def old_login(username, md5_password):
from api import agent_header
exponent = int("010001", 16)
modulus = int("AC69F5CCC8BDE47CD3D371603748378C9CFAD2938A6B021E0E191013975AD683F5CBF9ADE8BD7D46B4D2EC2D78A"
"F146F1DD2D50DC51446BB8880B8CE88D476694DFC60594393BEEFAA16F5DBCEBE22F89D640F5336E42F587DC4AF"
"EDEFEAC36CF007009CCCE5C1ACB4FF06FBA69802A8085C2C54BADD0597FC83E6870F1E36FD", 16)
param = '{"cmdID":1,"isCompressed":0,"rsaKey":{"n":"AC69F5CCC8BDE47CD3D371603748378C9CFAD2938A6B0' \
'21E0E191013975AD683F5CBF9ADE8BD7D46B4D2EC2D78AF146F1DD2D50DC51446BB8880B8CE88D476694DFC60594393BEEFAA16F' \
'5DBCEBE22F89D640F5336E42F587DC4AFEDEFEAC36CF007009CCCE5C1ACB4FF06FBA69802A8085C2C54BADD0597FC83E6870F1E3' \
'6FD","e":"010001"},"businessType":%s,"passWord":"******","loginType":0,"sdkVersion":177588,' \
'"appName":"ANDROID-com.xunlei.redcrystalandroid","platformVersion":1,"devicesign":"%s",' \
'"sessionID":"","protocolVersion":%s,"userName":"******","extensionList":"","sequenceNo":%s,' \
'"peerID":"","clientVersion":"1.0.0"}'
_chars = "0123456789ABCDEF"
deviceid = username
device_id = md5(deviceid)
appName = 'com.xunlei.redcrystalandroid'
businessType = '61'
key = 'C2049664-1E4A-4E1C-A475-977F0E207C9C'
key_md5 = md5(key)
device_sign = "div100.%s%s" % (device_id, md5(sha1("%s%s%s%s" % (device_id, appName, businessType, key_md5))))
hash_password = hex(pow_mod(StrToInt(md5_password), exponent, modulus))[2:].upper().zfill(256)
params = param % (61, hash_password, device_sign, 108, username, 1000006)
r = requests.post("https://login.mobile.reg2t.sandai.net/", data=params, headers=agent_header, verify=False)
login_status = json.loads(r.text)
return login_status
def continuity(url):
from util import md5
format = '%25s: %s'
# first fetch the file with the normal http handler
opener = urllib2.build_opener()
urllib2.install_opener(opener)
fo = urllib2.urlopen(url)
foo = fo.read()
fo.close()
m = md5(foo)
print format % ('normal urllib', m.hexdigest())
# now install the keepalive handler and try again
opener = urllib2.build_opener(HTTPHandler())
urllib2.install_opener(opener)
fo = urllib2.urlopen(url)
foo = fo.read()
fo.close()
m = md5(foo)
print format % ('keepalive read', m.hexdigest())
fo = urllib2.urlopen(url)
foo = ''
while True:
f = fo.readline()
if f:
foo = foo + f
else: break
fo.close()
m = md5(foo)
print format % ('keepalive readline', m.hexdigest())
def old_login(username, md5_password):
hash_password = rsa_encode(md5_password)
_chars = "0123456789ABCDEF"
peer_id = ''.join(random.sample(_chars, 16))
device_id = md5("%s23333" % md5_password) # just generate a 32bit string
appName = 'com.xunlei.redcrystalandroid'
key = 'C2049664-1E4A-4E1C-A475-977F0E207C9C'
md5_key = md5(key)
device_sign = 'div100.%s%s' % (
device_id, md5(sha1("%s%s%s%s" % (device_id, appName, 61, md5_key))))
payload = json.dumps({
"protocolVersion": PROTOCOL_VERSION,
"sequenceNo": 1000001,
"platformVersion": 1,
"sdkVersion": 177588,
"peerID": peer_id,
"businessType": 61,
"clientVersion": APP_VERSION,
"devicesign": device_sign,
"isCompressed": 0,
"cmdID": 1,
"userName": username,
"passWord": hash_password,
"loginType": 0,
"sessionID": "",
"verifyKey": "",
"verifyCode": "",
"appName": "ANDROID-com.xunlei.redcrystalandroid",
"rsaKey": {
"e": "%06X" % rsa_pubexp,
"n": long2hex(rsa_mod)
},
"extensionList": ""
})
headers = {
'User-Agent':
"Mozilla/5.0 (iPhone; CPU iPhone OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Mobile/9B176 MicroMessenger/4.3.2"
}
r = requests.post("https://login.mobile.reg2t.sandai.net/",
data=payload,
headers=headers,
verify=False)
login_status = json.loads(r.text)
return login_status
def collect_tweets_by_search_terms(search_configs_filepath, output_folder,
config):
apikeys = list(config['apikeys'].values()).pop()
search_configs = {}
with open(os.path.abspath(search_configs_filepath),
'r') as search_configs_rf:
search_configs = json.load(search_configs_rf)
for search_config_id in itertools.cycle(search_configs):
search_config = search_configs[search_config_id]
search_terms = [term.lower() for term in search_config['terms']]
querystring = '%s' % (' OR '.join('(' + term + ')'
for term in search_terms))
since_id = search_config[
'since_id'] if 'since_id' in search_config else 0
geocode = tuple(search_config['geocode']) if (
'geocode' in search_config and search_config['geocode']) else None
logger.info(
'REQUEST -> (md5(querystring): [%s]; since_id: [%d]; geocode: [%s])'
% (util.md5(querystring.encode('utf-8')), since_id, geocode))
try:
twitterCralwer = TwitterCrawler(apikeys=apikeys,
client_args=CLIENT_ARGS,
output_folder=output_folder)
since_id = twitterCralwer.search_by_query(querystring,
geocode=geocode,
since_id=since_id)
except Exception as exc:
logger.error(exc)
logger.error(util.full_stack())
pass
search_config['since_id'] = since_id
search_config['querystring'] = querystring
search_config['geocode'] = geocode
search_configs[search_config_id] = search_config
flash_cmd_config(search_configs, search_configs_filepath,
output_folder)
logger.info(
'COMPLETED -> (md5(querystring): [%s]; since_id: [%d]; geocode: [%s])'
% (util.md5(querystring.encode('utf-8')), since_id, geocode))
logger.info('PAUSE %ds to CONTINUE...' % WAIT_TIME)
time.sleep(WAIT_TIME)
def upload_image():
if 'file' not in request.files:
raise HttpError(400, "缺少file参数")
file = request.files['file']
if file.filename == '':
raise HttpError(400, "未发现文件")
content = file.read()
# 图片格式限制
extname = file.filename.rsplit('.', 1)[1].lower()
if extname not in ['png', 'jpg', 'jpeg', 'gif']:
raise HttpError(400, "不支持此格式图片上传")
# 图片大小限制
max_size = 5
size = int(math.ceil(len(content) / 1024.0)) # size KB
if size > max_size * 1024:
raise HttpError(400, u"图片不能超过%sM" % max_size)
md5sum = md5(content).lower()
full_path = "%s/%s/%s/%s.%s" % (app.config["IMAGE_UPLOAD_PATH"],
md5sum[:2], md5sum[2:4], md5sum, extname)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
output_file = open(full_path, 'w+')
output_file.write(content)
url = path_to_url(full_path)
res = rpc.invbox.check_add_image(md5sum,
url,
base_url=app.config["DOMAIN"])
return jsonify(res)
def get_page_comments(self,shortcode,delay=DELAY,count=-1,save=False,path=None,tname=None):
results = []
_count = 0
page = self.get_page_info(shortcode)
comment_card = page['graphql']['shortcode_media']['edge_media_to_comment']
total = comment_card['count']
page_info = comment_card['page_info']
top_comments = comment_card['edges']
end_cursor = page_info['end_cursor']
has_next = page_info['has_next_page']
headers = COMMON_HEADERS
headers['x-ig-app-id']=self.app_id
headers['referer'] = API_PICTURE_PAGE.format(shortcode=shortcode)
_check = count if count > 0 else total
for i in top_comments:
if save:
self.db.save(i,tname=tname)
results.append(i)
_count += 1
if (_count >= count or _count >= total) and (count > 0):
logger.info(f'[Done]Get crawled comments of page:"{shortcode}":{len(results)}.[Total({total})]')
return results
if not has_next:
logger.info(f'[Done]Get crawled comments of page:"{shortcode}":{len(results)}.[Total({total})]')
return results
while 1:
if not end_cursor:
logger.info(f'[Done]Get crawled comments of page:"{shortcode}":{len(results)}.[Total({total})]')
break
params = copy.deepcopy(COMMENTS_PARAMS)
params['query_hash']=self.comment_hash
params['variables']=params['variables'].replace('$',end_cursor).replace('%',shortcode)
md5ed = md5(self.rhx_gis + ":" + params['variables'])
headers['x-instagram-gis']=md5ed
response = send_request(API_USER_POSTS,
params=params,
headers=headers,
delay=delay,
proxy=PROXY_GLOBAL,
json=True)
json_data = response.json()
data = json_data['data']['shortcode_media']['edge_media_to_comment']['edges']
page_info = json_data['data']['shortcode_media']['edge_media_to_comment']['page_info']
for i in data:
if save:
self.db.save(i,tname=tname)
results.append(i)
_count += 1
if (_count >= count or _count >= total) and (count > 0):
logger.info(f'[Done]Get crawled comments of page:"{shortcode}"'
f':{len(results)}.[Total({total})]')
return results
logger.info(f'Current crawled comments of page "{shortcode}"'
f':{len(results)}.[{round(len(results)/_check,4)*100 if _check else 0}%]')
end_cursor = page_info['end_cursor']
if not page_info['has_next_page']:
logger.info(f'[Done]Get crawled comments of page:"{shortcode}"'
f':{len(results)}.[Total({total})]')
break
return results
def _hash(git_url: str) -> str:
"""
hash the git url as a unique key
:param git_url: git url
:return: md5 hash of git url
"""
return util.md5(git_url)
def init_base():
users = {
"luke": "13064754229",
"april": "13621713595",
}
for name, mobile in users.items():
admin = M.Admin.create(
mobile=mobile,
username=name,
password=md5(mobile[-6:]),
)
admin.save()
user = M.User.create(
mobile=mobile,
username=name,
)
user.save()
# apkversion
apk = M.ApkVersion.create(version="1.0.0",
up_type=1,
url="/media/apk/smallbox.apk")
apk.save()
def test_mapping_dist(metasync, opts):
mapping = [("dropbox", 2), ("google", 15), ("box", 10), ("onedrive", 7), ("baidu", 2048)]
mapping = map(lambda x:(util.md5(x[0])%10000,x[1]), mapping)
print(mapping)
hspace = (2+15+10+7+2048)*5
objs = []
with open("result/linux_objs.txt") as f:
for line in f:
sp = line.strip().split("\t")
hv = sp[0]
size = int(sp[1])
objs.append( (hv, size) )
for replication in range(1, 4):
detmap = DetMap2(mapping, hspace, replication)
sizes = {}
counts = {}
for srv, sz in mapping:
sizes[srv] = 0
counts[srv] = 0
for obj in objs:
hv = obj[0]
size = obj[1]
lst = detmap.get_mapping(hv)
for srv in lst:
counts[srv] += 1
sizes[srv] += size
print replication,
for srv, sz in mapping:
print "%d/%d" % (counts[srv],sizes[srv]),
print
def make_post_params(self, offset=0):
#限制时间范围
post_params = {
'sdate': self.sdate_str,
'edate': self.edate_str,
'_t': int(time.time())
}
#限制统计类型
if self.pm.statistics:
post_params['type'] = ','.join(self.pm.statistics)
if self.pm.is_create_time:
post_params['is_create_time'] = 1
#限制条目数
post_params['offset'] = offset
post_params['limit'] = self.LIMIT_NUM
#设置签名
post_params_str = urllib.urlencode(post_params)
app_key = self.platform_model.app_key
sign = md5('%s%s' % (post_params_str, app_key))
post_params_str = '%s&sign=%s' % (post_params_str, sign)
return post_params_str
def __init__(self, url, save_path=None, status_func=None, done_func=None,
debug=False):
self.url = url
self.debug = debug
if save_path is None:
save_path = url.split('/')[-1]
if save_path[0] != '/':
save_path = path.join(os.getcwd(), save_path)
self.save_path = save_path
self.hash = util.md5(url)
self.temp_dir = path.join(os.getcwd(), 'tmp')
self.data_path = path.join(self.temp_dir, self.hash)
self.meta = {}
self.task_queue = multiprocessing.Queue()
self.process_num = 10
if hasattr(status_func, '__call__'):
status_func("status")
if hasattr(done_func, '__call__'):
done_func('done')
self.start_time = time.time()
def create_sign(data, extra_code):
#print("-----------")
#print(data)
dict_keys = data.keys()
#print(dict_keys)
keys = list(dict_keys)
keys.sort()
#jjprint(keys)
#print("-----------")
#print [key for key in keys]
#print [data[key] for key in keys]
message = ""
for k in keys:
value = data[k]
if k != "sign":
if k != "param":
#print(value)
message = message + value
else:
tmp = json.dumps(value,separators=(',',':'))
message = message + tmp
message = message + extra_code
print("message=%s" %message)
sign = md5(message.encode("utf-8"))
#print sign
return sign
def __init__(self,
url,
save_path=None,
status_func=None,
done_func=None,
process_num=10,
debug=False):
self.url = url
self.debug = debug
if save_path is None:
save_path = url.split('/')[-1]
if save_path[0] != '/':
save_path = os.path.join(os.getcwd(), save_path)
self.save_path = save_path
self.hash = util.md5(url)
self.temp_dir = os.path.join(os.getcwd(), 'tmp')
self.data_path = os.path.join(self.temp_dir, self.hash)
self.meta = {}
self.task_queue = multiprocessing.Queue()
self.process_num = 10
if hasattr(status_func, '__call__'):
status_func("status")
if hasattr(done_func, '__call__'):
done_func('done')
self.start_time = time.time()
def succeed(self, url, detail_urls, batch_id):
crawl_time = time.time()
self.logger.info("parse %s successful" % url)
sql_url = "UPDATE list_url SET crawl_time=FROM_UNIXTIME(%d), crawl_status=0, status=3, status_time=now() " \
"WHERE url_md5='%s' AND batch_id=%d AND UNIX_TIMESTAMP(crawl_time) <= %d" % \
(crawl_time, util.md5(url), batch_id, crawl_time)
sql_client = "UPDATE client_list SET crawl_time=FROM_UNIXTIME(%d), crawl_num=crawl_num+1 "\
"WHERE id=%d" % (crawl_time, self.id)
sql_task_batch = "UPDATE task_batch SET last_crawl_time=FROM_UNIXTIME(%d), crawl_num=crawl_num+1, "\
"parse_url_num=parse_url_num+%d WHERE task_type='%s' and batch_id=%d" % \
(crawl_time, len(detail_urls), self.task_type, batch_id)
self.db_operator(sql_url)
self.db_operator(sql_client)
self.db_operator(sql_task_batch)
table_name = "detail_urls_%s" % time.strftime(
"%Y%m%d", time.localtime(crawl_time))
sql_create = """CREATE table IF NOT EXISTS %s (
`id` int(11) NOT NULL auto_increment,
`url` text NOT NULL,
`list_url` text NOT NULL,
`task_type` varchar(20) NOT NULL,
`client_id` int(11) NOT NULL,
`batch_id` int(11) NOT NULL default 1,
`update_time` timestamp NOT NULL default CURRENT_TIMESTAMP,
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8""" % table_name
self.db_operator(sql_create)
sql_insert = "INSERT INTO %s(url, list_url, task_type, client_id, batch_id) VALUES" % table_name
values = ",".join(["('%s', '%s', '%s', %d, %d)" % \
(detail_url, url, self.task_type, self.id, batch_id) for detail_url in detail_urls])
sql_insert += values
self.db_operator(sql_insert)
def verify_file():
filename = request.args.get('filename')
checksum = request.args.get('checksum')
filename = secure_filename(filename)
basename = os.path.splitext(filename)[0]
stagingdir = os.path.join(cfg.STAGING_FOLDER, basename)
stagingpath = os.path.join(stagingdir, filename)
if not os.path.exists(stagingpath):
log.error('File %s does not exist', stagingpath)
raise Error(message=('File %s does not exist' % stagingpath),
status_code=404)
calculated_checksum = util.md5(stagingpath)
valid = calculated_checksum == checksum
if valid:
log.info('File %s successfully verified', filename)
return util.ret_ok()
else:
log.error('File %s: hash mismatch. Given: %s, calculated: %s',
filename, checksum, calculated_checksum)
raise Error(message=('File hash mismatch. Given: %s, calculated: %s' %
(checksum, calculated_checksum)),
status_code=400)
def test_mapping_dist(metasync, opts):
mapping = [("dropbox", 2), ("google", 15), ("box", 10), ("onedrive", 7),
("baidu", 2048)]
mapping = map(lambda x: (util.md5(x[0]) % 10000, x[1]), mapping)
print(mapping)
hspace = (2 + 15 + 10 + 7 + 2048) * 5
objs = []
with open("result/linux_objs.txt") as f:
for line in f:
sp = line.strip().split("\t")
hv = sp[0]
size = int(sp[1])
objs.append((hv, size))
for replication in range(1, 4):
detmap = DetMap2(mapping, hspace, replication)
sizes = {}
counts = {}
for srv, sz in mapping:
sizes[srv] = 0
counts[srv] = 0
for obj in objs:
hv = obj[0]
size = obj[1]
lst = detmap.get_mapping(hv)
for srv in lst:
counts[srv] += 1
sizes[srv] += size
print replication,
for srv, sz in mapping:
print "%d/%d" % (counts[srv], sizes[srv]),
print
def get_posts(self,delay=DELAY,count=-1,save=False,path=None,tname=None):
_count = 0
results = []
_check = count if count > 0 else self.posts_count
top_posts_card = self.info['edge_owner_to_timeline_media']
top_posts = top_posts_card['edges']
end_cursor = top_posts_card['page_info']['end_cursor']
posts_query_id = self.queryIds[2]
headers = COMMON_HEADERS
headers['x-ig-app-id']=self.app_id
for i in top_posts:
if save:
self.db.save(i,tname=tname)
_count += 1
results.append(i)
if (_count >= count or _count >= self.posts_count) and (count > 0):
logger.info(f'[Done]The length of crawled data of user "{self.name}"'
f':{len(results)}.[Total({self.posts_count})]')
return results
logger.info(f'Total posts of user "{self.name}":{self.posts_count}.')
while 1:
if not end_cursor:
logger.info(f'[Done]The length of crawled data of user "{self.name}"'
f':{len(results)}.[Total({self.posts_count})]')
break
params = {}
params['query_hash']=posts_query_id
params['variables']=r'{"id":"'+self.Id+'","first":"'+\
str(USER_POSTS_MAX)+'","after":"'+end_cursor+'"}'
md5ed = md5(self.rhx_gis + ":" + params['variables'])
headers['x-instagram-gis']=md5ed
response = send_request(API_USER_POSTS,
params=params,
headers=headers,
delay=delay,
json=True,
proxy=PROXY_GLOBAL)
json_data = response.json()
data = json_data['data']['user']\
['edge_owner_to_timeline_media']['edges']
page_info = json_data['data']['user']\
['edge_owner_to_timeline_media']['page_info']
for i in data:
if save:
self.db.save(i,tname=tname)
results.append(i)
_count += 1
if (_count >= count or _count >= self.posts_count) and (count > 0):
logger.info(f'[Done]The length of crawled data of user "{self.name}"'
f':{len(results)}.[Total({self.posts_count})]')
return results
logger.info(f'Current amount of posts of user "{self.name}"'
f':{len(results)}.[{round(len(results)/_check,4)*100 if _check else 0}%]')
end_cursor = page_info['end_cursor']
if not page_info['has_next_page']:
logger.info(f'[Done]The length of crawled data of user "{self.name}"'
f':{len(results)}.[Total({self.posts_count})]')
break
return results
def post_headers(agentConfig, payload):
return {
'User-Agent': 'Datadog Agent/%s' % agentConfig['version'],
'Content-Type': 'application/json',
'Content-Encoding': 'deflate',
'Accept': 'text/html, */*',
'Content-MD5': md5(payload).hexdigest()
}
def old_login(username, md5_password):
hash_password = rsa_encode(md5_password)
_chars = "0123456789ABCDEF"
peer_id = ''.join(random.sample(_chars, 16))
device_id = md5("%s23333" % md5_password) # just generate a 32bit string
appName = 'com.xunlei.redcrystalandroid'
key = 'C2049664-1E4A-4E1C-A475-977F0E207C9C'
md5_key = md5(key)
device_sign = 'div100.%s%s' % (device_id, md5(sha1("%s%s%s%s" % (device_id, appName, 61, md5_key))))
payload = json.dumps({
"protocolVersion": PROTOCOL_VERSION,
"sequenceNo": 1000001,
"platformVersion": 1,
"sdkVersion": 177588,
"peerID": peer_id,
"businessType": 61,
"clientVersion": APP_VERSION,
"devicesign": device_sign,
"isCompressed": 0,
"cmdID": 1,
"userName": username,
"passWord": hash_password,
"loginType": 0,
"sessionID": "",
"verifyKey": "",
"verifyCode": "",
"appName": "ANDROID-com.xunlei.redcrystalandroid",
"rsaKey": {
"e": "%06X" % rsa_pubexp,
"n": long2hex(rsa_mod)
},
"extensionList": ""
})
headers = {'User-Agent': "Mozilla/5.0 (iPhone; CPU iPhone OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Mobile/9B176 MicroMessenger/4.3.2"}
r = requests.post("https://login.mobile.reg2t.sandai.net/", data=payload, headers=headers, verify=False)
login_status = json.loads(r.text)
return login_status
def post_headers(agentConfig, payload):
return {
'User-Agent': 'Datadog Agent/%s' % agentConfig['version'],
'Content-Type': 'application/json',
'Content-Encoding': 'deflate',
'Accept': 'text/html, */*',
'Content-MD5': md5(payload).hexdigest()
}
def get_channel_posts(self,delay=DELAY,count=-1,save=False,path=None,tname=None):
_count = 0
results = []
_check = count if count > 0 else self.channel_posts_count
top_posts_card = self.info['edge_felix_video_timeline']
top_posts = top_posts_card['edges']
end_cursor = top_posts_card['page_info']['end_cursor']
headers = COMMON_HEADERS
headers['x-ig-app-id'] = self.app_id
for i in top_posts:
if save:
self.db.save(i,tname=tname)
_count += 1
results.append(i)
if (_count >= count or _count >= self.channel_posts_count) and (count > 0):
logger.info(f'[Done]The amount of crawled channel posts data of user "{self.name}":{len(results)}.'
f'[Total({self.channel_posts_count})]')
return results
logger.info(f'Total channel posts of user "{self.name}":{self.channel_posts_count}.')
while 1:
if not end_cursor:
logger.info(f'[Done]The amount of crawled channel posts data of user "{self.name}":{len(results)}.'
f'[Total({self.channel_posts_count})]')
break
params = copy.deepcopy(CHANNEL_PARAMS)
params['variables'] = params['variables'].replace('%',self.Id).replace('$',end_cursor)
params['query_hash'] = self.channel_hash
md5ed = md5(self.rhx_gis + ":" + params['variables'])
headers['x-instagram-gis']=md5ed
response = send_request(API_USER_POSTS,
session=self.instagram.session,
params=params,
headers=headers,
delay=delay,
json=True,
proxy=PROXY_GLOBAL)
json_data = response.json()
posts = json_data['data']['user']['edge_felix_video_timeline']['edges']
page_info = json_data['data']['user']['edge_felix_video_timeline']['page_info']
has_next_page = page_info['has_next_page']
end_cursor = page_info['end_cursor']
for i in posts:
if save:
self.db.save(i,tname=tname)
results.append(i)
_count += 1
if (_count >= count or _count >= self.channel_posts_count) and (count > 0):
logger.info(f'[Done]The amount of crawled channel posts data of user "{self.name}"'
f':{len(results)}.[Total({self.channel_posts_count})]')
return results
logger.info(f'Current amount of crawled channel posts data of user "{self.name}"'
f':{len(results)}.[{round(len(results)/_check,4)*100 if _check else 0}%]')
if not has_next_page:
logger.info(f'[Done]The amount of crawled channel posts data of user "{self.name}"'
f':{len(results)}.[Total({self.channel_posts_count})]')
break
return results
def testSpeed(self):
# Pretend to be gmetad and serve a large piece of content
server = subprocess.Popen("nc -l 8651 < %s" % TEST_FN, shell=True)
# Wait for 1 second
time.sleep(1)
pfile = tempfile.NamedTemporaryFile()
g = Ganglia(logging.getLogger(__file__))
# Running the profiler
# profile.runctx("g.check({'ganglia_host': 'localhost', 'ganglia_port': 8651})", {}, {"g": g}, pfile.name)
# p = pstats.Stats(pfile.name)
# p.sort_stats('time').print_stats()
self.assertEquals(
md5(g.check({
'ganglia_host': 'localhost',
'ganglia_port': 8651
})).hexdigest(),
md5(open(TEST_FN).read()).hexdigest())
def updateFiles(self, destination, filegroup):
"""
Updates the files in a given file group, in the destination subdirectory of the Forged Alliance path.
If existing=True, the existing contents of the directory will be added to the current self.filesToUpdate
list.
"""
QtWidgets.QApplication.processEvents()
self.progress.setLabelText("Updating files: " + filegroup)
self.destination = destination
self.connection.writeToServer("GET_FILES_TO_UPDATE", filegroup)
self.waitForFileList()
# Ensure our list is unique
self.filesToUpdate = list(set(self.filesToUpdate))
targetdir = os.path.join(util.APPDATA_DIR, destination)
if not os.path.exists(targetdir):
os.makedirs(targetdir)
for fileToUpdate in self.filesToUpdate:
md5File = util.md5(
os.path.join(util.APPDATA_DIR, destination, fileToUpdate))
if md5File is None:
if self.version:
if self.featured_mod == "faf" or self.featured_mod == "ladder1v1" or \
filegroup == "FAF" or filegroup == "FAFGAMEDATA":
self.connection.writeToServer("REQUEST_VERSION",
destination,
fileToUpdate,
str(self.version))
else:
self.connection.writeToServer(
"REQUEST_MOD_VERSION", destination, fileToUpdate,
json.dumps(self.modversions))
else:
self.connection.writeToServer("REQUEST_PATH", destination,
fileToUpdate)
else:
if self.version:
if self.featured_mod == "faf" or self.featured_mod == "ladder1v1" or \
filegroup == "FAF" or filegroup == "FAFGAMEDATA":
self.connection.writeToServer("PATCH_TO", destination,
fileToUpdate, md5File,
str(self.version))
else:
self.connection.writeToServer(
"MOD_PATCH_TO", destination, fileToUpdate, md5File,
json.dumps(self.modversions))
else:
self.connection.writeToServer("UPDATE", destination,
fileToUpdate, md5File)
self.waitUntilFilesAreUpdated()
def old_login(username, md5_password):
from api import agent_header
exponent = int("010001", 16)
modulus = int(
"AC69F5CCC8BDE47CD3D371603748378C9CFAD2938A6B021E0E191013975AD683F5CBF9ADE8BD7D46B4D2EC2D78A"
"F146F1DD2D50DC51446BB8880B8CE88D476694DFC60594393BEEFAA16F5DBCEBE22F89D640F5336E42F587DC4AF"
"EDEFEAC36CF007009CCCE5C1ACB4FF06FBA69802A8085C2C54BADD0597FC83E6870F1E36FD",
16)
param = '{"cmdID":1,"isCompressed":0,"rsaKey":{"n":"AC69F5CCC8BDE47CD3D371603748378C9CFAD2938A6B0' \
'21E0E191013975AD683F5CBF9ADE8BD7D46B4D2EC2D78AF146F1DD2D50DC51446BB8880B8CE88D476694DFC60594393BEEFAA16F' \
'5DBCEBE22F89D640F5336E42F587DC4AFEDEFEAC36CF007009CCCE5C1ACB4FF06FBA69802A8085C2C54BADD0597FC83E6870F1E3' \
'6FD","e":"010001"},"businessType":%s,"passWord":"******","loginType":0,"sdkVersion":177588,' \
'"appName":"ANDROID-com.xunlei.redcrystalandroid","platformVersion":1,"devicesign":"%s",' \
'"sessionID":"","protocolVersion":%s,"userName":"******","extensionList":"","sequenceNo":%s,' \
'"peerID":"","clientVersion":"1.0.0"}'
_chars = "0123456789ABCDEF"
deviceid = username
device_id = md5(deviceid)
appName = 'com.xunlei.redcrystalandroid'
businessType = '61'
key = 'C2049664-1E4A-4E1C-A475-977F0E207C9C'
key_md5 = md5(key)
device_sign = "div100.%s%s" % (
device_id,
md5(sha1("%s%s%s%s" % (device_id, appName, businessType, key_md5))))
hash_password = hex(pow_mod(StrToInt(md5_password), exponent,
modulus))[2:].upper().zfill(256)
params = param % (61, hash_password, device_sign, 108, username, 1000006)
r = requests.post("https://login.mobile.reg2t.sandai.net/",
data=params,
headers=agent_header,
verify=False)
login_status = json.loads(r.text)
return login_status
def legalFAVersion(self):
#Now we check the FA version
luascd = os.path.join(self.path, 'gamedata', 'lua.scd')
if not os.path.isfile(luascd):
return False
md5LUA = util.md5(luascd)
log("lua.scd digest is %s" % md5LUA)
return md5LUA in ["4af45c46b0abb2805bea192e6e2517d4","5cdd99bddafa38f0873bd11dd353055a","ad999839d32f4784e331e5e8be1d32a2"]
def account_add():
user = session.get('user_info')
account_name = request.values.get('xl_username')
password = request.values.get('xl_password')
md5_password = md5(password)
user_key = '%s:%s' % ('user', user.get('username'))
user_info = json.loads(r_session.get(user_key).decode('utf-8'))
if user_info.get('max_account_no') is None:
user_info['max_account_no'] = 1
accounts_key = 'accounts:%s' % user.get('username')
account_no = r_session.scard(accounts_key)
if account_no is not None:
if account_no >= user_info.get('max_account_no'):
session['error_message'] = '你的账号限制%d个账户。' % account_no
return redirect(url_for('accounts'))
login_result = login(account_name, md5_password,
app.config.get('ENCRYPT_PWD_URL'))
if login_result.get('errorCode') != 0:
error_message = login_result.get('errorDesc')
session['error_message'] = '登陆失败，错误信息：%s。' % error_message
return redirect(url_for('accounts'))
xl_session_id = login_result.get('sessionID')
xl_nick_name = login_result.get('nickName')
xl_user_name = login_result.get('userName')
xl_user_id = str(login_result.get('userID'))
xl_user_new_no = str(login_result.get('userNewNo'))
xl_account_name = account_name
xl_password = md5_password
r_session.sadd(accounts_key, xl_user_id)
account_key = 'account:%s:%s' % (user.get('username'), xl_user_id)
xl_account_data = dict(
session_id=xl_session_id,
nick_name=xl_nick_name,
username=xl_user_name,
user_id=xl_user_id,
user_new_no=xl_user_new_no,
account_name=xl_account_name,
password=xl_password,
active=True,
status='OK',
createdtime=datetime.now().strftime('%Y-%m-%d %H:%M'))
r_session.set(account_key, json.dumps(xl_account_data))
return redirect(url_for('accounts'))
def get_best_results(self, n_top_rows):
by = self.get_result_column_index()
df = pd.DataFrame(self.data)
df = df.sort_values(by=by, ascending=False)
df['md5'] = df.apply(lambda x: md5(str(tuple(x))), axis=1)
#temporaly display all columns
with pd.option_context('display.max_rows', None):
print(df.head(n_top_rows))
return df.head(n_top_rows).values
def account_add():
session['action']='one'
account_name = request.values.get('xl_username')
password = request.values.get('xl_password')
md5_password = md5(password)
user = session.get('user_info')
accounts_key = 'accounts:%s' % user.get('username')
user_key = '%s:%s' % ('user', user.get('username'))
user_info = json.loads(r_session.get(user_key).decode('utf-8'))
account_no = r_session.scard(accounts_key) + 1
if user_info.get('account_limit') is not None and user_info['account_limit'] < account_no:
session['error_message']='账户数量已达上限，无法完成添加'
return redirect(url_for('accounts'))
if account_no is not None:
if account_no >= user_info.get('total_account_point'):
session['error_message']='账户余额不足，无法完成添加'
return redirect(url_for('accounts'))
elif account_no >= user_info.get('max_account_no'):
user_info['max_account_no'] = account_no
days=int(user_info.get('total_account_point') / user_info.get('max_account_no'))
if days<36500:
user_info['expire_date'] = (datetime.now() + timedelta(days=days)).strftime('%Y-%m-%d')
else:
user_info['expire_date'] = (datetime.now() + timedelta(days=36500)).strftime('%Y-%m-%d')
r_session.set(user_key, json.dumps(user_info))
login_result = login(account_name, md5_password, app.config.get('ENCRYPT_PWD_URL'))
if login_result.get('errorCode') != 0:
error_message = login_result.get('errorDesc')
session['error_message'] = '登陆失败，错误信息：%s。' % error_message
return redirect(url_for('accounts'))
xl_session_id = login_result.get('sessionID')
xl_nick_name = login_result.get('nickName')
xl_user_name = login_result.get('userName')
xl_user_id = str(login_result.get('userID'))
xl_user_new_no = str(login_result.get('userNewNo'))
xl_account_name = account_name
xl_password = md5_password
r_session.sadd(accounts_key, xl_user_id)
account_key = 'account:%s:%s' % (user.get('username'), xl_user_id)
xl_account_data = dict(session_id=xl_session_id, nick_name=xl_nick_name, username=xl_user_name,
user_id=xl_user_id, user_new_no=xl_user_new_no, account_name=xl_account_name,
password=xl_password, active=True, status='OK',
createdtime=datetime.now().strftime('%Y-%m-%d %H:%M'))
r_session.set(account_key, json.dumps(xl_account_data))
return redirect(url_for('accounts'))
def default_template_file_action(self,
dir_name,
file_name,
dest_rel_path=None,
dest_name=None):
template_full = dir_name + '/' + file_name
Config.log("default_template_file_action '%s'" % template_full,
tag='DEFAULT_TEMPLATE_FILE_ACTION')
if dest_name:
rel_path = dest_rel_path
dest_path = u.pathify(self.output_root, dest_rel_path)
else:
rel_path = u.make_rel_path(self.site_root, dir_name)
dest_path = u.pathify(self.output_root, rel_path)
dest_name = file_name
u.ensure_path(dest_path)
dest_full = u.pathify(dest_path, dest_name)
info = {
'name': dest_name,
'path': dest_path,
'rel_path': rel_path,
'full': dest_full,
'key': u.make_key(rel_path, dest_name)
}
if self.config.is_template_type(file_name):
template = open(template_full).read()
output = u.debracket(template, self.interpret)
if not self.config.is_special_file(info['key']):
open(dest_full, 'w').write(output)
local = u.local_metadata(dest_path, dest_name)
info['size'] = local['size']
info['modified'] = local['modified']
info['md5'] = u.md5(dest_full)
self.track_file(info)
else:
shutil.copyfile(template_full, dest_full)
local = u.local_metadata(dest_path, dest_name)
info['size'] = local['size']
info['modified'] = local['modified']
info['md5'] = u.md5(dest_full)
self.track_file(info)
def get_by_client(self, task_type, resp):
if not self.conn: return 0
if not resp: return 0
item = json.loads(resp)
url = item['url'].encode("utf-8", "ignore")
_id = util.md5(url)[:24]
try:
self.conn[task_type].update({'_id': ObjectId(_id)},
{'$set': {'Status': 2, 'StatusTime': datetime.datetime.now()}},
multi = True)
except Exception, e:
print 'error:', type(url), _id
def _load(self, config):
nspace = []
for n, size in config:
for s in range(size):
nspace.append((n,s))
assert len(nspace) < self.hspace
self.detmap = [None] * self.hspace
for i in range(self.hspace):
group = []
for n in nspace:
order = int(util.md5("%s.%s" % (i, n)))
group.append((n[0], order))
self.detmap[i] = sorted(group, key=lambda e:e[1])
def testSpeed(self):
# Pretend to be gmetad and serve a large piece of content
server = subprocess.Popen("nc -l 8651 < %s" % TEST_FN, shell=True)
# Wait for 1 second
time.sleep(1)
pfile = tempfile.NamedTemporaryFile()
g = Ganglia(logging.getLogger(__file__))
# Running the profiler
# profile.runctx("g.check({'ganglia_host': 'localhost', 'ganglia_port': 8651})", {}, {"g": g}, pfile.name)
# p = pstats.Stats(pfile.name)
# p.sort_stats('time').print_stats()
self.assertEquals(md5(g.check({'ganglia_host': 'localhost', 'ganglia_port': 8651})).hexdigest(), md5(open(TEST_FN).read()).hexdigest())
def _load(self, config):
nspace = []
for n, size in config:
for s in range(size):
nspace.append((n, s))
assert len(nspace) < self.hspace
self.detmap = [None] * self.hspace
for i in range(self.hspace):
group = []
for n in nspace:
order = int(util.md5("%s.%s" % (i, n)))
group.append((n[0], order))
self.detmap[i] = sorted(group, key=lambda e: e[1])
def cache_view(self,request, view_func, view_args, view_kwargs):
'''页面级别的缓存
'''
cache_method_list = ['']
_mc = MemcachedUtil()
params = str(request.REQUEST)
view_func_name = view_func.__name__
key = md5('%s_%s_%s' % (request.admin.id,view_func.__name__,params))
response = _mc.get(key)
if not response:
response = view_func(request,*view_args,**view_kwargs)
_mc.set(key, response, 1800)
return response
def geo_search(self, call='query', query=None):
if not query:
raise Exception("geo_search: query cannot be empty")
now = datetime.datetime.now()
day_output_folder = os.path.abspath(
'%s/%s' % (self.output_folder, now.strftime('%Y%m%d')))
if not os.path.exists(day_output_folder):
os.makedirs(day_output_folder)
filename = os.path.abspath(
'%s/%s' % (day_output_folder, util.md5(query.encode('utf-8'))))
retry_cnt = MAX_RETRY_CNT
while retry_cnt > 0:
try:
result = None
if ('query' == call):
result = self.search_geo(query=query)
elif ('ip' == call):
result = self.search_geo(ip=query)
else:
logger.error("call == ('query' or 'ip')")
return
if (result):
with open(filename, 'a+', newline='',
encoding='utf-8') as f:
f.write('%s\n' % json.dumps(result))
time.sleep(1)
return False
except twython.exceptions.TwythonRateLimitError:
self.rate_limit_error_occured('geo', '/geo/search')
except Exception as exc:
time.sleep(10)
logger.error("exception: %s; when fetching place: %s" %
(exc, query))
retry_cnt -= 1
if (retry_cnt == 0):
logger.warn("exceed max retry... return")
return
return
def check(self, instance):
url = instance['url']
default_timeout = self.init_config.get('default_timeout', 5)
timeout = float(instance.get('timeout', default_timeout))
aggregation_key = md5(url).hexdigest()
try:
h = Http(timeout=timeout)
resp, content = h.request(url, "GET")
except socket.timeout, e:
self.timeout_event(url, timeout, aggregation_key)
return
def parse_failed(self, url, batch_id):
self.logger.error("parse %s failed" % url)
crawl_time = time.time()
sql_url = "UPDATE list_url SET crawl_time=FROM_UNIXTIME(%d), crawl_status=2, status=3, status_time=now() " \
"WHERE url_md5='%s' AND batch_id=%d, UNIX_TIMESTAMP(crawl_time) <= %d" % \
(crawl_time, util.md5(url), batch_id, crawl_time)
sql_client = "UPDATE client_list SET crawl_time=FROM_UNIXTIME(%d), crawl_num=crawl_num+1, " \
"parse_failed_num=parse_failed_num+1 WHERE id=%d" % (crawl_time, self.id)
sql_task_batch = "UPDATE task_batch SET last_crawl_time=FROM_UNIXTIME(%d), crawl_num=crawl_num+1, "\
"parse_failed_num=parse_failed_num+1 "\
"WHERE task_type='%s' and batch_id=%d" % (crawl_time, self.task_type, batch_id)
self.db_operator(sql_url)
self.db_operator(sql_client)
self.db_operator(sql_task_batch)
def check(self, instance):
url = instance['url']
default_timeout = self.init_config.get('default_timeout', 5)
timeout = float(instance.get('timeout', default_timeout))
aggregation_key = md5(url).hexdigest()
try:
h = Http(timeout=timeout)
resp, content = h.request(url, "GET")
except socket.timeout, e:
self.timeout_event(url, timeout, aggregation_key)
return
def _upload(self, src_path, src_name, key, bucket=None, content_type=None, extra_args=None):
full = u.pathify(src_path, src_name)
if extra_args is None:
extra_args = {
'Metadata': {
'md5': u.md5(full)
}
}
if not content_type:
content_type = mimetypes.MimeTypes().guess_type(src_name)[0]
extra_args['ContentType'] = content_type
dest_full = self._file_dest_root + '/' + key
dest_dir, dest_name = os.path.split(dest_full)
u.ensure_path(dest_dir)
shutil.copyfile(full, dest_full)
def geo_search(self, call='query', query=None):
if not query:
raise Exception("geo_search: query cannot be empty")
now=datetime.datetime.now()
day_output_folder = os.path.abspath('%s/%s'%(self.output_folder, now.strftime('%Y%m%d')))
if not os.path.exists(day_output_folder):
os.makedirs(day_output_folder)
filename = os.path.abspath('%s/%s'%(day_output_folder, util.md5(query.encode('utf-8'))))
retry_cnt = MAX_RETRY_CNT
while retry_cnt > 0:
try:
result = None
if ('query' == call):
result = self.search_geo(query=query)
elif ('ip' == call):
result = self.search_geo(ip=query)
else:
logger.error("call == ('query' or 'ip')")
return
if (result):
with open(filename, 'a+', newline='', encoding='utf-8') as f:
f.write('%s\n'%json.dumps(result))
time.sleep(1)
return False
except twython.exceptions.TwythonRateLimitError:
self.rate_limit_error_occured('geo', '/geo/search')
except Exception as exc:
time.sleep(10)
logger.error("exception: %s; when fetching place: %s"%(exc, query))
retry_cnt -= 1
if (retry_cnt == 0):
logger.warn("exceed max retry... return")
return
return
def account_add():
# 获取表单提交的迅雷会员名和帐户密码
account_name = request.values.get('xl_username')
password = request.values.get('xl_password')
md5_password = md5(password)
user = session.get('user_info')
accounts_key = 'accounts:%s' % user.get('username')
if user.get('max_account_no') is None:
user['max_account_no'] = 5
account_no = r_session.scard(accounts_key)
if account_no is not None:
if account_no >= user.get('max_account_no'):
session['error_message'] = '你的账号限制%d个账户。' % account_no
return redirect(url_for('accounts'))
login_result = login(account_name, md5_password,
app.config.get('ENCRYPT_PWD_URL'))
if login_result.get('errorCode') != 0:
error_message = login_result.get('errorDesc')
session['error_message'] = '登陆失败，错误信息：%s。' % error_message
return redirect(url_for('accounts'))
xl_session_id = login_result.get('sessionID')
xl_nick_name = login_result.get('nickName')
xl_user_name = login_result.get('userName')
xl_user_id = str(login_result.get('userID'))
xl_user_new_no = str(login_result.get('userNewNo'))
xl_account_name = account_name
xl_password = md5_password
r_session.sadd(accounts_key, xl_user_id)
account_key = 'account:%s:%s' % (user.get('username'), xl_user_id)
xl_account_data = dict(session_id=xl_session_id, nick_name=xl_nick_name, username=xl_user_name,
user_id=xl_user_id, user_new_no=xl_user_new_no, account_name=xl_account_name,
password=xl_password, active=True, status='OK',
createdtime=datetime.now().strftime('%Y-%m-%d %H:%M'))
r_session.set(account_key, json.dumps(xl_account_data))
return redirect(url_for('accounts'))
def updateFiles(self, destination, filegroup):
"""
Updates the files in a given file group, in the destination subdirectory of the Forged Alliance path.
If existing=True, the existing contents of the directory will be added to the current self.filesToUpdate
list.
"""
QtGui.QApplication.processEvents()
self.progress.setLabelText("Updating files: " + filegroup)
self.destination = destination
self.writeToServer("GET_FILES_TO_UPDATE", filegroup)
self.waitForFileList()
#Ensure our list is unique
self.filesToUpdate = list(set(self.filesToUpdate))
targetdir = os.path.join(util.APPDATA_DIR, destination)
if not os.path.exists(targetdir):
os.makedirs(targetdir)
for fileToUpdate in self.filesToUpdate:
md5File = util.md5(os.path.join(util.APPDATA_DIR, destination, fileToUpdate))
if md5File == None:
if self.version:
if self.featured_mod == "faf" or self.featured_mod == "ladder1v1" or filegroup == "FAF" or filegroup == "FAFGAMEDATA":
self.writeToServer("REQUEST_VERSION", destination, fileToUpdate, str(self.version))
else:
self.writeToServer("REQUEST_MOD_VERSION", destination, fileToUpdate,
json.dumps(self.modversions))
else:
self.writeToServer("REQUEST_PATH", destination, fileToUpdate)
else:
if self.version:
if self.featured_mod == "faf" or self.featured_mod == "ladder1v1" or filegroup == "FAF" or filegroup == "FAFGAMEDATA":
self.writeToServer("PATCH_TO", destination, fileToUpdate, md5File, str(self.version))
else:
self.writeToServer("MOD_PATCH_TO", destination, fileToUpdate, md5File,
json.dumps(self.modversions))
else:
self.writeToServer("UPDATE", destination, fileToUpdate, md5File)
self.waitUntilFilesAreUpdated()
def get(self):
form = forms.BookmarkletForm(self)
if form.validate():
if not form.title.data:
form.title.data = form.url.data
url_digest = util.md5(form.url.data)
bookmark = self.db.bookmarks.find_one({
'user': self.current_user._id,
'url_digest': url_digest})
if bookmark is None:
bookmark = {'user': self.current_user._id,
'modified': datetime.datetime.now()}
bookmark = tornado.web._O(bookmark)
form.populate_obj(bookmark)
self.db.bookmarks.save(bookmark)
self.write('oldu')
else:
self.write('%s' % form.errors)
def download_textures(self):
self.progress.show()
self.progress.setLabelText("Downloading resources ...")
textInCache = []
for tex in self.texturelist :
if os.path.exists(self.get_texture_name(tex)) :
if util.md5(self.get_texture_name(tex)) == self.texturelist[tex] :
#logger.debug(tex + ".png in cache.")
textInCache.append(tex)
continue
#logger.debug("Downloading " + tex + ".png")
self.downloader.get(QNetworkRequest(QtCore.QUrl(TEXTURE_SERVER + tex + ".png")))
for tex in textInCache :
del self.texturelist[tex]
if len(self.texturelist) == 0 :
self.progress.close()
self.setup()
def sid(self):
return util.md5("baidu") % 10000
def sid(self):
return util.md5("google") % 10000
def search_by_query(self, query, since_id = 0, geocode=None, lang=None, output_filename = None):
if not query:
raise Exception("search: query cannot be None")
now=datetime.datetime.now()
#logger.info("query: %s; since_id: %d"%(query, since_id))
place = None
geo = None
if (geocode):
place, geo = geocode
day_output_folder = os.path.abspath('%s/%s/%s'%(self.output_folder, now.strftime('%Y%m%d'), place))
else:
day_output_folder = os.path.abspath('%s/%s'%(self.output_folder, now.strftime('%Y%m%d')))
if not os.path.exists(day_output_folder):
os.makedirs(day_output_folder)
filename = os.path.abspath('%s/%s'%(day_output_folder, util.md5(query.encode('utf-8'))))
prev_max_id = -1
current_max_id = 0
cnt = 0
current_since_id = since_id
retry_cnt = MAX_RETRY_CNT
#result_tweets = []
while current_max_id != prev_max_id and retry_cnt > 0:
try:
if current_max_id > 0:
tweets = self.search(q=query, geocode=geo, since_id=since_id, lang=lang, tweet_mode='extended', max_id=current_max_id-1, result_type='recent', count=100)
else:
tweets = self.search(q=query, geocode=geo, since_id=since_id, lang=lang, tweet_mode='extended', result_type='recent', count=100)
prev_max_id = current_max_id # if no new tweets are found, the prev_max_id will be the same as current_max_id
with open(filename, 'a+', newline='', encoding='utf-8') as f:
for tweet in tweets['statuses']:
f.write('%s\n'%json.dumps(tweet))
if current_max_id == 0 or current_max_id > int(tweet['id']):
current_max_id = int(tweet['id'])
if current_since_id == 0 or current_since_id < int(tweet['id']):
current_since_id = int(tweet['id'])
#no new tweets found
if (prev_max_id == current_max_id):
break
#result_tweets.extend(tweets['statuses'])
cnt += len(tweets['statuses'])
# if (cnt % 1000 == 0):
# logger.info("[%d] tweets... "%cnt)
#logger.info(cnt)
#logger.debug('%d > %d ? %s'%(prev_max_id, current_max_id, bool(prev_max_id > current_max_id)))
time.sleep(1)
except twython.exceptions.TwythonRateLimitError:
self.rate_limit_error_occured('search', '/search/tweets')
except Exception as exc:
time.sleep(10)
logger.error("exception: %s"%exc)
retry_cnt -= 1
if (retry_cnt == 0):
logger.warn("exceed max retry... return")
return since_id
#raise MaxRetryReached("max retry reached due to %s"%(exc))
logger.info("[%s]; since_id: [%d]; total tweets: %d "%(query, since_id, cnt))
return current_since_id
def collect_tweets_by_search_terms(search_configs_filepath, output_folder, config):
apikeys = list(config['apikeys'].values()).pop()
search_configs = {}
with open(os.path.abspath(search_configs_filepath), 'r') as search_configs_rf:
search_configs = json.load(search_configs_rf)
for search_config_id in itertools.cycle(search_configs):
search_config = search_configs[search_config_id]
search_terms = [term.lower() for term in search_config['terms']]
querystring = '%s'%(' OR '.join('(' + term + ')' for term in search_terms))
since_id = search_config['since_id'] if 'since_id' in search_config else 0
geocode = tuple(search_config['geocode']) if ('geocode' in search_config and search_config['geocode']) else None
logger.info('REQUEST -> (md5(querystring): [%s]; since_id: [%d]; geocode: [%s])'%(util.md5(querystring.encode('utf-8')), since_id, geocode))
try:
twitterCralwer = TwitterCrawler(apikeys=apikeys, client_args=CLIENT_ARGS, output_folder = output_folder)
since_id = twitterCralwer.search_by_query(querystring, geocode = geocode, since_id = since_id)
except Exception as exc:
logger.error(exc)
logger.error(util.full_stack())
pass
search_config['since_id'] = since_id
search_config['querystring'] = querystring
search_config['geocode'] = geocode
search_configs[search_config_id] = search_config
flash_cmd_config(search_configs, search_configs_filepath, output_folder)
logger.info('COMPLETED -> (md5(querystring): [%s]; since_id: [%d]; geocode: [%s])'%(util.md5(querystring.encode('utf-8')), since_id, geocode))
logger.info('PAUSE %ds to CONTINUE...'%WAIT_TIME)
time.sleep(WAIT_TIME)
def sid(self):
return util.md5("onedrive") % 10000
def __init__(self, root):
self.root = root
util.mkdirs(root)
self._sid = util.md5(self.root) % 10000
def sid(self):
return util.md5("dropbox") % 10000