def getlist(url):
try:
html = urllib2.urlopen(url).read()
reg = '''<a href="(.*)" target="_blank" rel="nofollow">
<h2 class="question-title">(.*)</h2></br></a>'''
reg = re.compile(reg)
return re.findall(reg,html)
except Exception as e:
error_list.append(url)
time.spleep(2)
def test_buscar_xpath(self):
driver = self.driver
driver.get("http://www.gmail.com")
time.sleep(3)
buscar_por_xpath = driver.find_element_by_xpath(
"//*[@id='identifierId']")
time.sleep(3)
buscar_por_xpath.send_keys("*****@*****.**")
buscar_por_xpath.send_keys(Keys.ENTER)
buscar_por_xpath = driver.find_element_by_xpath(
"//*[@id='password']/div[1]/div/div[1]/input")
time.sleep(3)
buscar_por_xpath.send_keys("*****@*****.**")
buscar_por_xpath.send_keys(Keys.ENTER)
time.spleep(3)
def build_bucket(user):
user_email = user.email
try:
local_bucket = userBucket.objects.get(ownerMail = user_email)
except Exception as err:
acl = 'private'
headers = {}
new_bucket = user.id + '--'+ time.strftime("%Y-%b-%d%H-%M-%S").lower()
hashed_bucket = hashlib.new("md5",new_bucket).hexdigest()
res = oss.put_bucket(hashed_bucket, acl, headers)
if (res.status / 100) == 2:
local_bucket = userBucket(ownerMail=user_email,bucket=new_bucket)
local_bucket.save()
else:
print "put bucket ", bucket, "ERROR"
time.spleep(1)
build_bucket(user)
return False
return True
def htmlprint(body):
print("Content-type: text/html; charset=utf-8\n")
html = f"""
<!DOCTYPE html>
<html lang="fr">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title> Anime </title>
</head>
<body>
<div class="list" id="style-1">
{body}
"""
time.spleep(5)
html += """
</div>
</body>
</html>
"""
print(html)
def stop_ec2_instance(ec2_con_re_for_region,my_instance_id):
pr_st=get_status_of_ec2(ec2_con_re_for_region,my_instance_id)
if pr_st=="stopped":
print"already it is stopped"
else:
filter1={"Name":"instance-id","Values":[my_instance_id]}
for each in ec2_con_re_for_region.instances.filter(Filters=[filter1]):
print dir(each)
each.stop()
'''
try:
each.wait_until_stopped()
except Exception as e:
print e
''' while True:
time.spleep(60)
pr_st=get_status_of_ec2(ec2_con_re_for_region,my_instance_id)
if pr_st=="stopped":
break
print"your instance is stopped"
def download_data(cell):
n_replicates = 2
characteristics_dict = {}
exp_open_chrom = ["FAIRE-seq", "ATAC-seq", "DNase-seq", "MNase-seq"]
exp_meth_dna = ["MRE-seq", "TAB-seq", "WGBS"]
chipseq = ["ChIP-seq"]
quant = ["polyA RNA-seq", "total RNA-seq"]
loop = ["ChIA-PET", "Hi-C", "5C"]
########### reading a list of known tfs in human
output = open(cell.replace(" ", "_") + ".txt", "w")
output.write("computing for " + cell + "\n")
characteristics_dict[cell.replace(" ", "_")] = {
"open_chromatin": {
"hg19": {
"G1": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"G2": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"none": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
}
},
"GRCh38": {
"G1": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"G2": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"none": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
}
}
},
"DNA_meth": {
"hg19": {
"G1": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"G2": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"none": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
}
},
"GRCh38": {
"G1": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"G2": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"none": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
}
}
},
"histone_marks": {
"hg19": {
"G1": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"G2": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"none": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
}
},
"GRCh38": {
"G1": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"G2": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"none": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
}
}
},
"TF_bound": {
"hg19": {
"G1": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"G2": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"none": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
}
},
"GRCh38": {
"G1": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"G2": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"none": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
}
}
},
"quantification": {
"hg19": {
"G1": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"G2": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"none": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
}
},
"GRCh38": {
"G1": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"G2": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"none": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
}
}
},
"loops": {
"hg19": {
"G1": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"G2": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"none": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
}
},
"GRCh38": {
"G1": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"G2": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
},
"none": {
"treatment": 0,
"genetic_mod": 0,
"none": 0
}
}
}
}
try:
URL = "https://www.encodeproject.org/search/?type=Experiment&replicates.library.biosample.donor.organism.scientific_name=H**o+sapiens&biosample_term_name=" + (
cell).replace(" ", "+") + "&limit=all"
# GET the object
response = None
while response == None:
try:
response = requests.get(URL, headers=HEADERS)
# Extract the JSON response as a python dict
response_json_dict = response.json()
i = 0
data = collections.defaultdict(dict)
while True:
try:
if len(response_json_dict["@graph"][i]["replicates"][0]
["biological_replicate_number"]
) >= n_replicates:
ID = response_json_dict["@graph"][i]["@id"]
URL2 = "https://www.encodeproject.org" + ID
# GET the object
response2 = None
while response2 == None:
try:
response2 = requests.get(URL2,
headers=HEADERS)
# Extract the JSON response as a python dict
response_json_dict2 = response2.json()
assay = "None"
try:
assay = response_json_dict2[
"assay_title"]
except:
try:
assay = response_json_dict2[
"assay_term_name"]
except:
pass
current_assembly = []
current_treatment = None
current_phase = "none"
current_genetic_modification = None
status = ""
#definition for keys to use
try:
current_assembly = response_json_dict2[
"assembly"]
except:
pass
try:
current_treatment = response_json_dict2[
"replicates"][0]["library"][
"biosample"]["treatments"]
except:
pass
try:
current_phase = response_json_dict2[
"replicates"][0]["library"][
"biosample"]["phase"]
except:
pass
try:
current_genetic_modification = response_json_dict2[
"replicates"][0]["library"][
"biosample"][
"genetic_modifications"]
except:
pass
try:
status = response_json_dict2[
"replicates"][0]["status"]
except:
pass
current_exp = False
if assay in exp_open_chrom:
current_exp = "open_chromatin"
elif assay in exp_meth_dna:
current_exp = "DNA_meth"
elif assay in chipseq:
#if pipeline is complete and status is released
#its time to know if it is an TF or a Histone mark
target = response_json_dict2["target"][
"name"].replace("-human", "")
current_exp = ""
if target in tfs: #if target is a TF
current_exp = "TF_bound"
elif target[
0] == "H": #maybe it is an histone mark
current_exp = "histone_marks"
elif assay in quant:
current_exp = "quantification"
elif assay in loop:
current_exp = "loops"
if status == "released" and current_exp != False and response_json_dict2[
"assembly"] != [] and (
response_json_dict2[
"internal_status"]
== "release ready"
): #or response_Sjson_dict2["internal_status"] == "pipeline completed" ):
for ass in current_assembly:
current_adding = "none"
if current_treatment != None:
current_adding = "treatment"
elif current_genetic_modification != None:
current_adding = "genetic_mod"
#print cell.replace(" ","_"), ass, current_phase, current_adding
characteristics_dict[cell.replace(
" ", "_")][current_exp][ass][
current_phase][
current_adding] += 1
except:
output.write(
"error trying to get response from " +
URL2 + " retrying in 10 seconds")
time.spleep(10)
except:
pass
i += 1
except:
output.write(
"Some error trying to read a response... waiting 10 second until to retry do get data from "
+ URL)
time.spleep(10)
except:
pass
output.write("Resume of " + cell + "\n")
output.write(
str(json.dumps(characteristics_dict, indent=4,
separators=(',', ': '))))
output.close()
print "##### end of calculations for " + cell + "cell line."
#!/usr/bin/python2
import time, os
print '''
Press 1: Show the current time
Press 2: Reboot the OS
Press 3: To create user in OS
'''
input1 = int(raw_input('Your choice : '))
if input1 == 1:
x = time.ctime()
x1 = x.split()[3]
print x1
elif input1 == 2:
print 'The system will reboot now...'
time.spleep(2)
os.system('reboot')
elif input1 == 3:
x = ['tom', 'jerry']
for i in x:
os.system('useradd ' + i)
print i + ' user is added.'
else:
print 'Wrong Choice'
def command_ayuda(m): cid = m.chat.id bot.send_chat_action(cid, 'typing') time.spleep(1) bot.send_message( cid, AYUDA)
def hello(event, context):
print("second update !")
time.spleep(4)
return "hello-world"
import som
print(som.FREQUENCIA)
som.doppler()
exit()
#r = subprocess.run(['free', '-h'], stdout=subprocess.PIPE)
r = run(['apt-get', 'install' , '-y', 'sl'], stdout=PIPE, stderr=PIPE)
if r.returncode !=0:
print('Deu merda!!!!')
#print(dir(r))
#print(r.stdout.decode('utf-8'))
exit()
letras = ['A', 'B', 'C', 'D']
print(random.randint(100, 999))
time.spleep(1)
print(random.choice(letras))
print(datetime.datetime.now())
hoje = datetime.datetime.now()
print(hoje.strftime('%d/%m/%Y'))
from settings import sql
import cymruwhois, time
cur = sql.cursor()
# get all ips where there is no asn
cur.execute("select ip from client where asn is Null")
ips = [x[0].split("/")[0] for x in cur.fetchall()]
cw = cymruwhois.Client(memcache_host=None)
success = False
while not success:
try:
lu = cw.lookupmany(ips)
success = True
except:
time.spleep(10)
for l in lu:
if l.asn:
cur.execute("select asn from asn where asn=%s" % l.asn)
r = cur.fetchone()
if not r:
cur.execute("insert into asn (asn,owner) values(%s,'%s')" %
(l.asn, l.owner))
cur.execute("update client set asn=%s where ip='%s' and asn is Null" %
(l.asn, l.ip))
sql.commit()
def f(i):
print('in f [%s]' % i)
while random.randint(0, 1):
time.spleep(0.6)
def crawl_onion(url, domain, port, type_service, message, crawler_config):
crawler_config['url'] = url
crawler_config['port'] = port
print('Launching Crawler: {}'.format(url))
r_cache.hset('metadata_crawler:{}'.format(splash_url), 'crawling_domain',
domain)
r_cache.hset('metadata_crawler:{}'.format(splash_url), 'started_time',
datetime.datetime.now().strftime("%Y/%m/%d - %H:%M.%S"))
retry = True
nb_retry = 0
while retry:
try:
r = requests.get(f'http://{splash_url}', timeout=30.0)
retry = False
except Exception:
# TODO: relaunch docker or send error message
nb_retry += 1
if nb_retry == 2:
crawlers.restart_splash_docker(splash_url, splash_name)
time.spleep(20)
if nb_retry == 6:
on_error_send_message_back_in_queue(type_service, domain,
message)
publisher.error('{} SPASH DOWN'.format(splash_url))
print('--------------------------------------')
print(' \033[91m DOCKER SPLASH DOWN\033[0m')
print(' {} DOWN'.format(splash_url))
r_cache.hset('metadata_crawler:{}'.format(splash_url),
'status', 'SPLASH DOWN')
nb_retry == 0
print(' \033[91m DOCKER SPLASH NOT AVAILABLE\033[0m')
print(' Retry({}) in 10 seconds'.format(nb_retry))
time.sleep(10)
if r.status_code == 200:
r_cache.hset('metadata_crawler:{}'.format(splash_url), 'status',
'Crawling')
# save config in cash
UUID = str(uuid.uuid4())
r_cache.set('crawler_request:{}'.format(UUID),
json.dumps(crawler_config))
process = subprocess.Popen(
["python", './torcrawler/tor_crawler.py', UUID],
stdout=subprocess.PIPE)
while process.poll() is None:
time.sleep(1)
if process.returncode == 0:
output = process.stdout.read().decode()
print(output)
# error: splash:Connection to proxy refused
if 'Connection to proxy refused' in output:
on_error_send_message_back_in_queue(type_service, domain,
message)
publisher.error(
'{} SPASH, PROXY DOWN OR BAD CONFIGURATION'.format(
splash_url))
print(
'------------------------------------------------------------------------'
)
print(' \033[91m SPLASH: Connection to proxy refused')
print('')
print(' PROXY DOWN OR BAD CONFIGURATION\033[0m'.
format(splash_url))
print(
'------------------------------------------------------------------------'
)
r_cache.hset('metadata_crawler:{}'.format(splash_url),
'status', 'Error')
exit(-2)
else:
crawlers.update_splash_manager_connection_status(True)
else:
print(process.stdout.read())
exit(-1)
else:
on_error_send_message_back_in_queue(type_service, domain, message)
print('--------------------------------------')
print(' \033[91m DOCKER SPLASH DOWN\033[0m')
print(' {} DOWN'.format(splash_url))
r_cache.hset('metadata_crawler:{}'.format(splash_url), 'status',
'Crawling')
exit(1)
def main():
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
opts, args = getopt(argv[1:], "c:tb:srahmg", ["correction=", "list-tips", "buy-tips=", "score", "credit", "achievement", "history", "my-tips", "get-subject"])
except:
print("option error.")
exit(1)
if (len(opts) == 0):
usage()
exit(0)
# try:
conn.connect(("10.19.252.191", 8585))
# except:
# print("Error connection.")
# exit(1)
index = 0
argument = list()
conn.settimeout(3)
for o, a in opts:
if o in ("-c", "--correction"):
index = 8
print("Get test...")
sendRequest(conn, makeRequest(index, user, *argument))
ret = ""
print("Testing...")
index = 7
try:
try:
msg = conn.recv(4096)
s = (msg.decode("ascii")).split("::")
cmd = ""
for t in s:
cmd = a + " " + t
ret += os.popen(cmd, 'r').read()
except:
ret = ""
conn.close()
time.spleep(2)
except:
print("error...")
ret = ""
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
conn.connect(("10.19.252.191", 8585))
conn.settimeout(3)
argument.append(ret)
elif o in ("-t", "--list-tips"):
index = 0
elif o in ("-m", "--my-tips"):
index = 1
elif o in ("-s", "--score"):
index = 3
elif o in ("-r", "--credit"):
index = 4
elif o in ("-h", "--history"):
index = 5
elif o in ("-g", "--get-subject"):
index = 9
elif o in ("-a", "--achievement"):
index = 6
elif o in ("-b", "--buy-tips"):
index = 2
try:
argument.append(a)
except:
print("Invalide tips")
conn.close()
exit(0)
print("sending...")
sendRequest(conn, makeRequest(index, user, *argument))
try:
msg = conn.recv(4096)
print(msg.decode("ascii"))
except:
print("end")
return (0)
from settings import sql
import cymruwhois,time
cur=sql.cursor()
# get all ips where there is no asn
cur.execute("select ip from client where asn is Null");
ips=[x[0].split("/")[0] for x in cur.fetchall()]
cw=cymruwhois.Client(memcache_host=None)
success=False
while not success:
try:
lu=cw.lookupmany(ips)
success=True
except:
time.spleep(10)
for l in lu:
if l.asn:
cur.execute("select asn from asn where asn=%s"%l.asn)
r=cur.fetchone()
if not r:
cur.execute("insert into asn (asn,owner) values(%s,'%s')"%(l.asn,l.owner))
cur.execute("update client set asn=%s where ip='%s' and asn is Null"%(l.asn,l.ip))
sql.commit()
