Beispiel #1
0
def getlist(url):
    try:
        html = urllib2.urlopen(url).read()
        reg = '''<a href="(.*)" target="_blank" rel="nofollow">
        <h2 class="question-title">(.*)</h2></br></a>'''
        reg =  re.compile(reg)
        return re.findall(reg,html)
    except Exception as e:
        error_list.append(url)
        time.spleep(2)
Beispiel #2
0
    def test_buscar_xpath(self):
        driver = self.driver
        driver.get("http://www.gmail.com")
        time.sleep(3)
        buscar_por_xpath = driver.find_element_by_xpath(
            "//*[@id='identifierId']")
        time.sleep(3)
        buscar_por_xpath.send_keys("*****@*****.**")
        buscar_por_xpath.send_keys(Keys.ENTER)

        buscar_por_xpath = driver.find_element_by_xpath(
            "//*[@id='password']/div[1]/div/div[1]/input")
        time.sleep(3)
        buscar_por_xpath.send_keys("*****@*****.**")
        buscar_por_xpath.send_keys(Keys.ENTER)
        time.spleep(3)
Beispiel #3
0
def build_bucket(user):
    user_email = user.email
    try:
        local_bucket =  userBucket.objects.get(ownerMail = user_email)
    except Exception as err:
        acl = 'private'
        headers = {}
        new_bucket = user.id + '--'+ time.strftime("%Y-%b-%d%H-%M-%S").lower()
        hashed_bucket = hashlib.new("md5",new_bucket).hexdigest()
        res = oss.put_bucket(hashed_bucket, acl, headers)
        if (res.status / 100) == 2:
            local_bucket = userBucket(ownerMail=user_email,bucket=new_bucket)
            local_bucket.save()
        else:
            print "put bucket ", bucket, "ERROR"
            time.spleep(1)
            build_bucket(user) 
            return False
    return True
Beispiel #4
0
def htmlprint(body):
    print("Content-type: text/html; charset=utf-8\n")
    html = f"""
    <!DOCTYPE html>
    <html lang="fr">
        <head>
            <meta charset="UTF-8">
            <meta name="viewport" content="width=device-width, initial-scale=1.0">
            <title> Anime </title>
        </head>
        <body>
            <div class="list" id="style-1">
                {body}
            """
    time.spleep(5)
    html += """
            </div>
        </body>
    </html>
    """
    print(html)
Beispiel #5
0
def stop_ec2_instance(ec2_con_re_for_region,my_instance_id):
	pr_st=get_status_of_ec2(ec2_con_re_for_region,my_instance_id)
	if pr_st=="stopped":
		print"already it is stopped"
	else:
            filter1={"Name":"instance-id","Values":[my_instance_id]}
            for each in ec2_con_re_for_region.instances.filter(Filters=[filter1]):
            	print dir(each)
                each.stop()
                '''
                try:
                	each.wait_until_stopped()
                except Exception as e:
                	print e 
'''				while True:
					time.spleep(60)
					pr_st=get_status_of_ec2(ec2_con_re_for_region,my_instance_id)
					if pr_st=="stopped":
						break

                print"your instance is stopped"
Beispiel #6
0
def download_data(cell):
    n_replicates = 2
    characteristics_dict = {}

    exp_open_chrom = ["FAIRE-seq", "ATAC-seq", "DNase-seq", "MNase-seq"]
    exp_meth_dna = ["MRE-seq", "TAB-seq", "WGBS"]
    chipseq = ["ChIP-seq"]
    quant = ["polyA RNA-seq", "total RNA-seq"]
    loop = ["ChIA-PET", "Hi-C", "5C"]

    ########### reading a  list of known tfs in human

    output = open(cell.replace(" ", "_") + ".txt", "w")
    output.write("computing for " + cell + "\n")
    characteristics_dict[cell.replace(" ", "_")] = {
        "open_chromatin": {
            "hg19": {
                "G1": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "G2": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "none": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                }
            },
            "GRCh38": {
                "G1": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "G2": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "none": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                }
            }
        },
        "DNA_meth": {
            "hg19": {
                "G1": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "G2": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "none": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                }
            },
            "GRCh38": {
                "G1": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "G2": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "none": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                }
            }
        },
        "histone_marks": {
            "hg19": {
                "G1": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "G2": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "none": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                }
            },
            "GRCh38": {
                "G1": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "G2": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "none": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                }
            }
        },
        "TF_bound": {
            "hg19": {
                "G1": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "G2": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "none": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                }
            },
            "GRCh38": {
                "G1": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "G2": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "none": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                }
            }
        },
        "quantification": {
            "hg19": {
                "G1": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "G2": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "none": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                }
            },
            "GRCh38": {
                "G1": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "G2": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "none": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                }
            }
        },
        "loops": {
            "hg19": {
                "G1": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "G2": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "none": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                }
            },
            "GRCh38": {
                "G1": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "G2": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                },
                "none": {
                    "treatment": 0,
                    "genetic_mod": 0,
                    "none": 0
                }
            }
        }
    }

    try:
        URL = "https://www.encodeproject.org/search/?type=Experiment&replicates.library.biosample.donor.organism.scientific_name=H**o+sapiens&biosample_term_name=" + (
            cell).replace(" ", "+") + "&limit=all"
        # GET the object
        response = None
        while response == None:
            try:
                response = requests.get(URL, headers=HEADERS)
                # Extract the JSON response as a python dict
                response_json_dict = response.json()

                i = 0
                data = collections.defaultdict(dict)
                while True:
                    try:
                        if len(response_json_dict["@graph"][i]["replicates"][0]
                               ["biological_replicate_number"]
                               ) >= n_replicates:
                            ID = response_json_dict["@graph"][i]["@id"]
                            URL2 = "https://www.encodeproject.org" + ID
                            # GET the object
                            response2 = None
                            while response2 == None:
                                try:
                                    response2 = requests.get(URL2,
                                                             headers=HEADERS)
                                    # Extract the JSON response as a python dict
                                    response_json_dict2 = response2.json()
                                    assay = "None"
                                    try:
                                        assay = response_json_dict2[
                                            "assay_title"]
                                    except:
                                        try:
                                            assay = response_json_dict2[
                                                "assay_term_name"]
                                        except:
                                            pass
                                    current_assembly = []
                                    current_treatment = None
                                    current_phase = "none"
                                    current_genetic_modification = None
                                    status = ""
                                    #definition for keys to use
                                    try:
                                        current_assembly = response_json_dict2[
                                            "assembly"]
                                    except:
                                        pass
                                    try:
                                        current_treatment = response_json_dict2[
                                            "replicates"][0]["library"][
                                                "biosample"]["treatments"]
                                    except:
                                        pass
                                    try:
                                        current_phase = response_json_dict2[
                                            "replicates"][0]["library"][
                                                "biosample"]["phase"]
                                    except:
                                        pass
                                    try:
                                        current_genetic_modification = response_json_dict2[
                                            "replicates"][0]["library"][
                                                "biosample"][
                                                    "genetic_modifications"]
                                    except:
                                        pass
                                    try:
                                        status = response_json_dict2[
                                            "replicates"][0]["status"]
                                    except:
                                        pass

                                    current_exp = False
                                    if assay in exp_open_chrom:
                                        current_exp = "open_chromatin"
                                    elif assay in exp_meth_dna:
                                        current_exp = "DNA_meth"
                                    elif assay in chipseq:

                                        #if pipeline is complete and status is released
                                        #its time to know if it is an TF or a Histone mark
                                        target = response_json_dict2["target"][
                                            "name"].replace("-human", "")

                                        current_exp = ""
                                        if target in tfs:  #if target is a TF
                                            current_exp = "TF_bound"
                                        elif target[
                                                0] == "H":  #maybe it is an histone mark
                                            current_exp = "histone_marks"

                                    elif assay in quant:
                                        current_exp = "quantification"
                                    elif assay in loop:
                                        current_exp = "loops"

                                    if status == "released" and current_exp != False and response_json_dict2[
                                            "assembly"] != [] and (
                                                response_json_dict2[
                                                    "internal_status"]
                                                == "release ready"
                                            ):  #or response_Sjson_dict2["internal_status"] == "pipeline completed" ):
                                        for ass in current_assembly:

                                            current_adding = "none"
                                            if current_treatment != None:
                                                current_adding = "treatment"
                                            elif current_genetic_modification != None:
                                                current_adding = "genetic_mod"
                                            #print cell.replace(" ","_"), ass, current_phase, current_adding
                                            characteristics_dict[cell.replace(
                                                " ", "_")][current_exp][ass][
                                                    current_phase][
                                                        current_adding] += 1

                                except:
                                    output.write(
                                        "error trying to get response from " +
                                        URL2 + " retrying in 10 seconds")
                                    time.spleep(10)
                    except:
                        pass
                    i += 1
            except:
                output.write(
                    "Some error trying to read a response... waiting 10 second until to retry do get data from "
                    + URL)
                time.spleep(10)

    except:
        pass
    output.write("Resume of " + cell + "\n")
    output.write(
        str(json.dumps(characteristics_dict, indent=4,
                       separators=(',', ': '))))
    output.close()
    print "##### end of calculations for " + cell + "cell line."
Beispiel #7
0
#!/usr/bin/python2

import time, os

print '''
Press 1: Show the current time
Press 2: Reboot the OS
Press 3: To create user in OS
'''

input1 = int(raw_input('Your choice : '))

if input1 == 1:
    x = time.ctime()
    x1 = x.split()[3]
    print x1

elif input1 == 2:
    print 'The system will reboot now...'
    time.spleep(2)
    os.system('reboot')

elif input1 == 3:
    x = ['tom', 'jerry']
    for i in x:
        os.system('useradd ' + i)
        print i + ' user is added.'

else:
    print 'Wrong Choice'
Beispiel #8
0
def command_ayuda(m):
   cid = m.chat.id
   bot.send_chat_action(cid, 'typing')
   time.spleep(1)
   bot.send_message( cid, AYUDA)
Beispiel #9
0
def hello(event, context):
    print("second update !")
    time.spleep(4)
    return "hello-world"
Beispiel #10
0
import som

print(som.FREQUENCIA)
som.doppler()

exit()
#r = subprocess.run(['free', '-h'], stdout=subprocess.PIPE)

r = run(['apt-get', 'install' , '-y', 'sl'], stdout=PIPE, stderr=PIPE)

if r.returncode !=0:
	print('Deu merda!!!!')


#print(dir(r))
#print(r.stdout.decode('utf-8'))

exit()
letras = ['A', 'B', 'C', 'D']

print(random.randint(100, 999))

time.spleep(1)

print(random.choice(letras))

print(datetime.datetime.now())

hoje = datetime.datetime.now()
print(hoje.strftime('%d/%m/%Y'))
Beispiel #11
0
from settings import sql
import cymruwhois, time

cur = sql.cursor()

# get all ips where there is no asn
cur.execute("select ip from client where asn is Null")
ips = [x[0].split("/")[0] for x in cur.fetchall()]
cw = cymruwhois.Client(memcache_host=None)
success = False
while not success:
    try:
        lu = cw.lookupmany(ips)
        success = True
    except:
        time.spleep(10)

for l in lu:
    if l.asn:
        cur.execute("select asn from asn where asn=%s" % l.asn)
        r = cur.fetchone()
        if not r:
            cur.execute("insert into asn (asn,owner) values(%s,'%s')" %
                        (l.asn, l.owner))
        cur.execute("update client set asn=%s where ip='%s' and asn is Null" %
                    (l.asn, l.ip))
sql.commit()
Beispiel #12
0
def f(i):
    print('in f [%s]' % i)
    while random.randint(0, 1):
        time.spleep(0.6)
Beispiel #13
0
def crawl_onion(url, domain, port, type_service, message, crawler_config):
    crawler_config['url'] = url
    crawler_config['port'] = port
    print('Launching Crawler: {}'.format(url))

    r_cache.hset('metadata_crawler:{}'.format(splash_url), 'crawling_domain',
                 domain)
    r_cache.hset('metadata_crawler:{}'.format(splash_url), 'started_time',
                 datetime.datetime.now().strftime("%Y/%m/%d  -  %H:%M.%S"))

    retry = True
    nb_retry = 0
    while retry:
        try:
            r = requests.get(f'http://{splash_url}', timeout=30.0)
            retry = False
        except Exception:
            # TODO: relaunch docker or send error message
            nb_retry += 1

            if nb_retry == 2:
                crawlers.restart_splash_docker(splash_url, splash_name)
                time.spleep(20)

            if nb_retry == 6:
                on_error_send_message_back_in_queue(type_service, domain,
                                                    message)
                publisher.error('{} SPASH DOWN'.format(splash_url))
                print('--------------------------------------')
                print('         \033[91m DOCKER SPLASH DOWN\033[0m')
                print('          {} DOWN'.format(splash_url))
                r_cache.hset('metadata_crawler:{}'.format(splash_url),
                             'status', 'SPLASH DOWN')
                nb_retry == 0

            print('         \033[91m DOCKER SPLASH NOT AVAILABLE\033[0m')
            print('          Retry({}) in 10 seconds'.format(nb_retry))
            time.sleep(10)

    if r.status_code == 200:
        r_cache.hset('metadata_crawler:{}'.format(splash_url), 'status',
                     'Crawling')
        # save config in cash
        UUID = str(uuid.uuid4())
        r_cache.set('crawler_request:{}'.format(UUID),
                    json.dumps(crawler_config))

        process = subprocess.Popen(
            ["python", './torcrawler/tor_crawler.py', UUID],
            stdout=subprocess.PIPE)
        while process.poll() is None:
            time.sleep(1)

        if process.returncode == 0:
            output = process.stdout.read().decode()
            print(output)
            # error: splash:Connection to proxy refused
            if 'Connection to proxy refused' in output:
                on_error_send_message_back_in_queue(type_service, domain,
                                                    message)
                publisher.error(
                    '{} SPASH, PROXY DOWN OR BAD CONFIGURATION'.format(
                        splash_url))
                print(
                    '------------------------------------------------------------------------'
                )
                print('         \033[91m SPLASH: Connection to proxy refused')
                print('')
                print('            PROXY DOWN OR BAD CONFIGURATION\033[0m'.
                      format(splash_url))
                print(
                    '------------------------------------------------------------------------'
                )
                r_cache.hset('metadata_crawler:{}'.format(splash_url),
                             'status', 'Error')
                exit(-2)
            else:
                crawlers.update_splash_manager_connection_status(True)
        else:
            print(process.stdout.read())
            exit(-1)
    else:
        on_error_send_message_back_in_queue(type_service, domain, message)
        print('--------------------------------------')
        print('         \033[91m DOCKER SPLASH DOWN\033[0m')
        print('          {} DOWN'.format(splash_url))
        r_cache.hset('metadata_crawler:{}'.format(splash_url), 'status',
                     'Crawling')
        exit(1)
Beispiel #14
0
def     main():
    conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        opts, args = getopt(argv[1:], "c:tb:srahmg", ["correction=", "list-tips", "buy-tips=", "score", "credit", "achievement", "history", "my-tips", "get-subject"])
    except:
        print("option error.")
        exit(1)
    if (len(opts) == 0):
        usage()
        exit(0)

#    try:
    conn.connect(("10.19.252.191", 8585))
#    except:
#        print("Error connection.")
#        exit(1)
    index = 0
    argument = list()
    conn.settimeout(3)
    for o, a in opts:
        if o in ("-c", "--correction"):
            index = 8
            print("Get test...")
            sendRequest(conn, makeRequest(index, user, *argument))
            ret = ""
            print("Testing...")
            index = 7
            try:
                try:
                    msg = conn.recv(4096)
                    s = (msg.decode("ascii")).split("::")
                    cmd = ""
                    for t in s:
                        cmd = a + " " + t
                        ret += os.popen(cmd, 'r').read()
                except:
                    ret = ""
                conn.close()
                time.spleep(2)
            except:
                print("error...")
                ret = ""
            conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            conn.connect(("10.19.252.191", 8585))
            conn.settimeout(3)
            argument.append(ret)
        elif o in ("-t", "--list-tips"):
            index = 0
        elif o in ("-m", "--my-tips"):
            index = 1
        elif o in ("-s", "--score"):
            index = 3
        elif o in ("-r", "--credit"):
            index = 4
        elif o in ("-h", "--history"):
            index = 5
        elif o in ("-g", "--get-subject"):
            index = 9
        elif o in ("-a", "--achievement"):
            index = 6
        elif o in ("-b", "--buy-tips"):
            index = 2
            try:
                argument.append(a)
            except:
                print("Invalide tips")
                conn.close()
                exit(0)
    print("sending...")
    sendRequest(conn, makeRequest(index, user, *argument))
    try:
        msg = conn.recv(4096)
        print(msg.decode("ascii"))
    except:
        print("end")
    return (0)
Beispiel #15
0
from settings import sql
import cymruwhois,time

cur=sql.cursor()

# get all ips where there is no asn
cur.execute("select ip from client where asn is Null");
ips=[x[0].split("/")[0] for x in cur.fetchall()]
cw=cymruwhois.Client(memcache_host=None)
success=False
while not success:
    try:
        lu=cw.lookupmany(ips)
        success=True
    except:
        time.spleep(10)

for l in lu:
    if l.asn:
        cur.execute("select asn from asn where asn=%s"%l.asn)
        r=cur.fetchone()
        if not r:
            cur.execute("insert into asn (asn,owner) values(%s,'%s')"%(l.asn,l.owner))
        cur.execute("update client set asn=%s where ip='%s' and asn is Null"%(l.asn,l.ip))
sql.commit()