def doRequest(self): d = Download(self.Url) if d.doRequest(): return 1 self.recs = d.getSOURCE() return 0
def __init__(self): """ Classe per gestire la copia dei dati da una forma /dataset/class1/ /dataset/class1/test To /train/class1 test/class2 """ #variabile dove si trova il dataset self.data_dir = "/Users/Eric/Desktop/eric/Programmazione/python/DeepLearning/data/knifey-spoony" #Url dove posso scaricare un dataset self.data_url = "https://github.com/Hvass-Labs/knifey-spoony/raw/master/knifey-spoony.tar.gz" #path della cartella di train self.train_dir = os.path.join(self.data_dir, "train/") #path della cartella di test self.test_dir = os.path.join(self.data_dir, "test/") #dimensione immagine self.image_size = 200 #canali immagine self.num_channels = 3 self.img_shape = [self.image_size, self.image_size, self.num_channels] self.img_size_flat = self.image_size * self.image_size * self.num_channels #numero di classi del dataset self.num_classes = 3 self.download = Download()
def isGoogleSearch(schema, ip): d = Download(schema + '://' + ip) if d.doRequest(): return False if Utility.containsGoogle(d.getSOURCE()): return True return False
def Save(email): """ Saves the youtube videos and handles errors""" try: Download(email) except Exception as e: SendEmail( email['from'], email['subject'], 'Something went wrong while downloading the ' + email['type'] + ' file: ' + email['url'] + '\n\nThe error was: ' + str(e)) return False
def doRequest(self): playerId = str(self.PlayerId) seasonType = self.SeasonType.replace(" ", "+") url = self.Url + "PlayerId=" + playerId + "&SeasonType=" + seasonType + "&League=" + self.LeagueId d = Download(url) if d.doRequest() == 1: return 1 self.recs = dumps(loads(d.getSOURCE())) return 0
def run(self): url = self.BASE_URL + self.SeasonId + self.BASE_URL_PART_3 + str( self.PageNumber) + self.BASE_URL_PART_5 d = Download(url) if d.doRequest(): # fail print 'ERROR: ' + self.SeasonId + '-' + str(self.PageNumber) else: utfstr2file( d.getSOURCE(), './data/' + self.SeasonId + '-' + str(self.PageNumber) + '.raw') return url
def getStats(self): d = Download(self.API) if d.doRequest(): return False res = [] j = loads(d.getSOURCE()) for item in j['resultSets'][1]['rowSet']: res.append(item[1:]) if len(res) == 0: return False else: return res
def main(): # print(sys.argv[0]) # print(type(sys.argv[1])) print('\t\t###########\n\t\t# WELCOME #\n\t\t###########\n') lat1 = float(sys.argv[1]) lon1 = float(sys.argv[2]) lat2 = float(sys.argv[3]) lon2 = float(sys.argv[4]) print('\tStart coordinate (%f, %f)' % (lat1, lon1)) print('\tEnd coordinate (%f, %f)' % (lat2, lon2)) print('\tStart searching ...\n') sc = Search(lat1, lat2, lon1, lon2) sc.searchLevels() picl = sc.qkll[-1] lod = len(sc.qkll) print('\tSearching complete ... \n') dl = Download() tl = list() print('\tDownloading images ...\n') if not os.path.exists('./temp/') : os.makedirs('./temp/') for qk in picl: dl.getUrlImage(qk) tl.append(Tile(qk)) print('\tDownloading complete ...\n') ts = TileSystem() pX1, pY1 = ts.latLongToPixelXY(sc.MinLatitude, sc.MinLongitude, lod) pX2, pY2 = ts.latLongToPixelXY(sc.MaxLatitude, sc.MaxLongitude, lod) print('\tStart merging ...\n') mg = Imerge(pX1, pX2, pY1, pY2, lod) for t in tl: mg.fillIm(t) print('\tMerging complete ...\n') fname = input('\tPlease give a name to the Image.\n\t\t') mg.saveFig(fname) f = open(fname, 'w') f.write('Start coordinate\n \t(%f, %f)\nEnd coordinate\n \t(%f, %f)' % (lat1, lon1, lat2, lon2)) if 'y' == input('\tRemove caches? y?\n\t\t') : filelist = [ f for f in os.listdir('./temp/') ] for f in filelist: os.remove(os.path.join('./temp/', f)) print('\t\t##########\n\t\t# DONE #\n\t\t##########\n')
def main(): ipList = getIps(config.IP_FILE) top = {} for ip in ipList: print ip start = int(round(time.time())) obj = Download("http://" + ip) if not obj.doRequest(): end = int(round(time.time())) top[ip] = end - start tmp = [(v, k) for k, v in top.iteritems()] topList = [] for item in sorted(tmp): topList.append(item[1]) lst2File(topList, config.TOP_IP_FILE)
def process_download_file(self, file_to_process): with open( "ProcessingFiles" + self.directory_separator + file_to_process, "r") as download_file: try: download_data = json.loads(download_file.read()) for f in sorted(download_data["Files"]): self.map_download_directories( f.replace(self.remote_directory_to_sync + "/", "")) for f in download_data["Files"]: for file_to_download in download_data["Files"][f]: Download(self.ftp_sync, f, file_to_download) except Exception as e: Logger("Error - Unable to download file: " + str(download_file) + ", " + str(e))
def __init__(self, lat1, lat2, lon1, lon2) : self.ts = TileSystem() self.dl = Download() self.qkll = list() if lat1 > lat2 : self.MinLatitude = lat1 self.MaxLatitude = lat2 else : self.MinLatitude = lat2 self.MaxLatitude = lat1 if lon1 < lon2 : self.MinLongitude = lon1 self.MaxLongitude = lon2 else : self.MinLongitude = lon2 self.MaxLongitude = lon1
def __init__(self, user, host, no_notify, verbose, interval, workflow_id=None): self.host = host self.user = user self.interval = interval self.cromwell = Cromwell(host=host) self.messenger = Messenger(self.user) self.no_notify = no_notify self.verbose = verbose self.workflow_id = workflow_id if user == "*": self.event_subscribers = [EmailNotification(self.cromwell), SystemTestDownload(), Download(self.cromwell.host), GATKDownload()] engine = create_engine("sqlite:///" + config.workflow_db) Base.metadata.bind = engine DBSession = sessionmaker() DBSession.bind = engine self.session = DBSession()
def getStats(self): d = Download(self.API) if d.doRequest(): return False res = [] j = loads(d.getSOURCE()) for item in j['resultSets'][0]['rowSet']: tmp = [] name = item[3] pos = item[5] if item[6] == 'null': height = 'None' else: height = item[6] if item[7] == " ": weight = 'None' else: weight = item[7] age = item[9] if item[10] == 'R' or item[10] == 'None' or item[10] == None: exp = 0 else: exp = item[10] tmp.append(name) tmp.append(pos) tmp.append(height) tmp.append(weight) tmp.append(age) tmp.append(exp) res.append(tmp) if len(res) == 0: return False else: return res
def main(): parser = argparse.ArgumentParser(description='Find all protein database entrys of specified taxon IDs and their descendants.' \ ' One taxID or a taxID input file must be provided. Peptide-Databases from NCBI or Uniprot can be used. User defined databases,' \ ' if header contain taxon IDs (e.g. OX=1111) or ncbi/uniprot accession IDs.') parser.add_argument( '-i', '--input', dest='input', default=None, help='TaxID input file: tabular file containing a column of NCBI' ' taxon IDs. Columns tab separated.') parser.add_argument('-c', '--column', dest='column', type=positive_integer, default=0, help='The column (zero-based) in the tabular ' 'file that contains Taxon IDs. Default = 0.') parser.add_argument( '-t', '--taxon', dest='taxon', type=positive_integer, nargs='+', action='append', help= 'NCBI taxon ID/s for database extraction. Multiple taxonIDs seperated by space.' ) parser.add_argument( '-d', '--database', dest='database', choices=['ncbi', 'uniprot', 'swissprot', 'trembl'], default='uniprot', help= 'Database choice for analysis or for download. Choices: ncbi, uniprot, tremble, swissprot. ' 'No download, if databases with original name are stored in same folder as option --path ' ) parser.add_argument( '-p', '--path', dest='path', default=None, help='Path to folder with all needed ' 'databases: taxdump.tar.gz (for all databases), prot.accession2taxid or prot.accession2taxid.gz and ' 'pdb.accession2taxid.gz (for ncbi databases). Optional: peptide_database named: nr/nr.gz, ' 'uniprot_trembl.fasta/uniprot_trembl.fasta.gz or uniprot_sprot.fasta/uniprot_sprot.fasta.gz' ' or uniprot.fasta./uniprot.fasta.gz') parser.add_argument( '-o', '--out', dest='out', default=None, help= "File name and direction of the result taxon specified peptide database. " "Default = /taxon_specified_db_DATE/taxon_specific_database.fasta") parser.add_argument( '-n', '--dbname', dest='dbname', default=None, help= "Database name and direction. If database is in other folder than --path or name deviates from standard names." ) parser.add_argument( '-l', '--level', dest='level', choices=[ 'species', 'section', 'genus', 'tribe', 'subfamily', 'family', 'superfamily', 'order', 'superorder', 'class', 'phylum', 'kingdom', 'superkingdom' ], default=None, help= 'Hierarchy level up in anchestral tree. Choices: species, section, genus, tribe, ' 'subfamily, family, superfamily, order, superorder, class, phylum, kingdom, superkingdom' ) parser.add_argument( '-z', '--no_descendants', dest='no_descendants', action='store_true', default=False, help= 'Select peptide database only by given taxon IDs, descendant taxons are excluded.' ) parser.add_argument( '-s', '--species', dest='species', action='store_true', default=False, help= 'Select peptide database only until taxonomic level "species", descendents from species are excluded.' ) parser.add_argument( '-r', '--non_redundant', dest='non_redundant', action='store_true', default=False, help= 'Makes the final database non redundant in regard to sequences, headers are concatenated.' ) parser.add_argument( '-u', '--threads', dest='threads', type=positive_integer, action="store", help= 'Number of threads for using multiprocessing. Default = number of cores.' ) parser.add_argument( '-x', '--reduce_header', dest='reduce_header', action='store_true', default=False, help= 'Reduce the long headers of NCBI entries to accession IDs. Use only for NCBI databases.' ) parser.add_argument('--version', action='version', version=('version ' + __version__)) parser.add_argument( '-v', '--verbose', dest='verbose', action='store_true', default=False, help= 'Verbose shows details about program progress and more information.') options = parser.parse_args() # url adresses for download: url_protaccession2taxID = 'https://ftp.ncbi.nlm.nih.gov/pub/taxonomy/accession2taxid/prot.accession2taxid.gz' url_protaccession2taxID_md5 = 'https://ftp.ncbi.nlm.nih.gov/pub/taxonomy/accession2taxid/prot.accession2taxid.gz.md5' url_pdbaccession2taxID = 'https://ftp.ncbi.nlm.nih.gov/pub/taxonomy/accession2taxid/pdb.accession2taxid.gz' url_pdbaccession2taxID_md5 = 'https://ftp.ncbi.nlm.nih.gov/pub/taxonomy/accession2taxid/pdb.accession2taxid.gz.md5' url_taxdump = 'ftp://ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdump.tar.gz' url_taxdump_md5 = 'ftp://ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdump.tar.gz.md5' url_database_ncbi = 'ftp://ftp.ncbi.nlm.nih.gov/blast/db/FASTA/nr.gz' url_database_md5_ncbi = 'ftp://ftp.ncbi.nlm.nih.gov/blast/db/FASTA/nr.gz.md5' url_database_swissprot = 'ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/complete/uniprot_sprot.fasta.gz' url_database_trembl = 'ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/complete/uniprot_trembl.fasta.gz' url_uniprot_metadata = 'ftp://ftp.expasy.org/databases/uniprot/current_release/knowledgebase/complete/RELEASE.metalink' db_dict_name = { 'ncbi': url_database_ncbi.split('/')[-1], 'uniprot': 'uniprot.fasta.gz', 'swissprot': url_database_swissprot.split('/')[-1], 'trembl': url_database_trembl.split('/')[-1] } # if not option out, a new folder with name taxon_database and date for result database and log file is created if options.out: output_path = Path.cwd() / options.out else: output_path = Output.createDir(Path.cwd()) logger = initialize_logger(output_path, options.verbose) for arg, value in sorted(vars(options).items()): logger.debug("Argument %s: %r", arg, value) logger.debug("Result database and log file are saved in direction %s" % output_path) # set path_to_db and database_folder for all user input variants # if options.path specified: folder to all databases (can be without protein DB if options.dbname) # if not exist, create folder with user defined name in option --path skip_check = False if options.path: database_folder = Path.cwd() / options.path path_to_db = database_folder / db_dict_name[options.database] # try open config file and read path to database folder, if no path option is entered # no config file, new database folder created else: try: path_to_main = Path(__file__, '..').resolve() with open(str(path_to_main) + "/tax2proteome.config", 'r') as config: database_folder = Path(config.readline().strip()) path_to_db = database_folder / db_dict_name[options.database] except FileNotFoundError: database_folder = Path.cwd() / ('databases_' + str(date.today())) path_to_db = database_folder / db_dict_name[options.database] try: database_folder.mkdir() prot_gz_b = prot_b = pdb_b = taxdump_b = db_gz_b = db_b = False skip_check = True logger.info("Downloaded databases are saved in direction %s" % database_folder) except FileExistsError: logger.debug( "Database folder %s already exists. Checking for content." % database_folder) except OSError: logger.exception( "No permission to create new database folder.", exc_info=True) exit(1) if not database_folder.exists(): try: database_folder.mkdir() logger.info( "New folder %s created. All needed database files will be downloaded and stored in this " "direction." % database_folder) prot_gz_b = prot_b = pdb_b = taxdump_b = db_gz_b = db_b = False skip_check = True except OSError: logger.exception( "Database folder %s does not exist and can not be created." % database_folder, exc_info=True) exit(1) # user given path to database # given path to database checked, if not exists quit. Check if DB is in uniprot or ncbi format if options.dbname: path_to_db = Path.cwd() / options.dbname db_b = Output.check_files_exist([path_to_db])[0] if not db_b: logger.error( "Given database %s does not exist. Enter correct path under option --dbname. Program quits." % path_to_db) exit(1) if not TestFile.test_uniprot(options.dbname): options.database = 'ncbi' # check database folder for content # check if all needed files in database folder: bool values _b: True = file exists and not downloaded again if not skip_check: taxdump_b, prot_gz_b, prot_b, pdb_b, db_gz_b, db_b = Output.check_files_exist( [ database_folder / url_taxdump.split('/')[-1], database_folder / url_protaccession2taxID.split('/')[-1], database_folder / 'prot.accession2taxid', database_folder / url_pdbaccession2taxID.split('/')[-1], path_to_db, path_to_db.parents[0] / path_to_db.stem ]) if db_b: path_to_db = path_to_db.parents[0] / path_to_db.stem if not taxdump_b: logger.warning( "File taxdump.tar.gz does not exist does not exist under the path %s and will be downloaded." % str(database_folder)) if not pdb_b and options.database == 'ncbi': logger.warning( "File pdb.accession2taxid.gz does not exist does not exist under the path %s and will be" " downloaded." % str(database_folder)) if not prot_gz_b and not prot_b and options.database == 'ncbi': logger.warning( "File prot.accession2taxid.gz does not exist does not exist under the path %s and will be" " downloaded." % str(database_folder)) if options.dbname is None and not db_b and not db_gz_b: logger.warning( "Database file %s does not exist does not exist under the path %s and will be downloaded." % (db_dict_name[options.database], str(database_folder))) # download taxdump file (best at the same day) if not taxdump_b: taxdump_md5 = read_ncbi_hash(url_taxdump_md5, logger) dwl_taxdb = Download(url_taxdump, database_folder / url_taxdump.split('/')[-1], taxdump_md5) dwl_taxdb.download() logger.debug('End download of taxdump.tar.gz') # download prot.accession2taxid.gz (only for ncbi) and check md5 hash if not prot_gz_b and not prot_b and options.database == 'ncbi': md5_hash = read_ncbi_hash(url_protaccession2taxID_md5, logger) dwl_protaccession = Download(url_protaccession2taxID, database_folder / url_protaccession2taxID.split('/')[-1], md5=md5_hash) dwl_protaccession.download() logger.debug( 'End download from %s to location %s.' % (url_protaccession2taxID, str(database_folder / url_protaccession2taxID.split('/')[-1]))) # download pdb.accession2taxid.gz (only for ncbi) and check md5 hash if not pdb_b and options.database == 'ncbi': md5_hash = read_ncbi_hash(url_pdbaccession2taxID_md5, logger) dwl_pdbaccession = Download(url_pdbaccession2taxID, database_folder / url_pdbaccession2taxID.split('/')[-1], md5=md5_hash) dwl_pdbaccession.download() logger.debug( 'End download from %s to location %s.' % (url_pdbaccession2taxID, str(database_folder / url_pdbaccession2taxID.split('/')[-1]))) # download peptide database and check md5 hash if not db_b and not db_gz_b: if options.database == 'ncbi': database_version_ncbi = 'ncbi ' + str(date) md5_hash = read_ncbi_hash(url_database_md5_ncbi, logger) dwl_db = Download(url_database_ncbi, database_folder / db_dict_name['ncbi'], md5=md5_hash) dwl_db.download() logger.debug("Databaseversion: %s" % database_version_ncbi) path_to_db = database_folder / db_dict_name['ncbi'] else: if options.database == 'swissprot' or options.database == 'uniprot': database_version_swissprot, hash_swissprot = read_uniprot_metadata( url_uniprot_metadata, db_dict_name['swissprot'], logger) logger.debug("Database version swissprot: %s " % database_version_swissprot) dwl_db_swiss = Download(url_database_swissprot, database_folder / db_dict_name['swissprot'], md5=hash_swissprot) dwl_db_swiss.download() path_to_db = database_folder / db_dict_name['swissprot'] if options.database == 'trembl' or options.database == 'uniprot': database_version_trembl, hash_trembl = read_uniprot_metadata( url_uniprot_metadata, db_dict_name['trembl'], logger) logger.debug("Databaseversion trembl: %s." % database_version_trembl) dwl_db_trembl = Download(url_database_trembl, database_folder / db_dict_name['trembl'], md5=hash_trembl) dwl_db_trembl.download() path_to_db = database_folder / db_dict_name['trembl'] # concetenate swissprot and trembl to uniprot file if options.database == 'uniprot': try: logger.debug( "Concatenate swissprot and trembl to uniprot database with name uniprot.fasta" ) with open(str(database_folder / db_dict_name['trembl']), 'ab') as trembl: with open( str(database_folder / db_dict_name['swissprot']), 'rb') as swissprot: shutil.copyfileobj(swissprot, trembl) # rename trembl to uniprot: Path(database_folder / db_dict_name['trembl']).rename( database_folder / db_dict_name['uniprot']) logger.debug("Uniprot database is now ready.") path_to_db = database_folder / db_dict_name['uniprot'] except FileNotFoundError: logger.exception( "Creation of uniprot database file out of trembl and swissprot file failed.", exc_info=True) exit(1) # create config file try: path_to_main = Path(__file__, '..').resolve() with open(str(path_to_main / "tax2proteome.config"), 'w') as config: config.write(str(database_folder) + '\n') except OSError: logger.debug('Can not create config file') # Read taxIDs from option -t and option -i if options.taxon: taxIDs = set( [taxID for taxonlist in options.taxon for taxID in taxonlist]) else: taxIDs = set() if options.input: try: with open(options.input, 'r') as inputFile: for i, line in enumerate(inputFile): fields = line.rstrip('\r\n').split('\t') if len(fields) >= abs(options.column): taxID = fields[options.column].strip() if taxID.isdigit(): taxIDs.add(int(taxID)) else: logger.error( 'Value %s in line %i of taxon input file is not a number. ' 'Right column number specified?' % (taxID, i)) continue else: logger.error( 'Column number is bigger as number of columns in taxon ID input file. ' 'Program continues without taxon IDs from input file.' ) except FileNotFoundError: logger.exception( 'Taxon ID input file does not exist under specified path.', exc_info=True) if not taxIDs: logger.error( 'No taxon ID given. Please check your input. Program quits. ') raise Exception('No taxon IDs.') exit(1) logger.debug('Given Tax-IDs: %s' % ' '.join(str(it) for it in taxIDs)) # Try load pre-builded taxonomy graph or built taxonomy graph now if not (database_folder / 'taxon_graph').is_file(): taxon_graph = TaxonGraph() logger.debug("Start building taxon graph.") taxon_graph.create_graph(database_folder / url_taxdump.split('/')[-1]) logger.debug("Taxon graph successfully build.") # save TaxonGraph to harddrive: with open(str(database_folder / 'taxon_graph'), 'wb') as handle: pickle.dump(taxon_graph, handle, protocol=pickle.HIGHEST_PROTOCOL) logger.debug('Safe taxon graph to location: %s' % str(database_folder / 'taxon_graph')) # load Taxon Graph else: try: logger.debug('Load taxon graph.') with open(str(database_folder / 'taxon_graph'), 'rb') as handle: taxon_graph = pickle.load(handle) except UnicodeDecodeError or EOFError: logger.exception( "Failed opening path to taxon graph / taxon_graph is corrupted. Delete %s file." % str(database_folder / 'taxon_graph')) exit(1) # adjusts the hierarchy level, if level does not exist, take next smaller level if options.level: logger.debug( "Start selection of next ancestor of level %s for all given taxIDs" % options.level) taxIDs = { taxon_graph.find_level_up(taxID, options.level) for taxID in taxIDs } logger.info( "All taxon IDs are set up to level %s in anchestral tree. Taxon IDs of level %s: %s" % (options.level, options.level, ' '.join(str(it) for it in taxIDs))) final_taxIDs = set() # find all descendants if not options.no_descendants: logger.debug("Start searching for all child taxon IDs.") for taxID in taxIDs: final_taxIDs.update(taxon_graph.find_taxIDs( taxID, options.species)) logger.debug("End searching for all child taxon IDs.") logger.debug('Number of final taxon IDs: %s' % str(len(final_taxIDs))) else: final_taxIDs = taxIDs logger.debug('Number of taxon IDs for database search: %s' % str(len(final_taxIDs))) # generate accession_taxID dict for ncbi db search and write custom specified db to --out with_taxon_ID = TestFile.test_uniprot(path_to_db) if not with_taxon_ID: accession = Accession(final_taxIDs) logger.debug('Read accession files.') if prot_b: accession.read_accessions( database_folder / 'prot.accession2taxid', database_folder / url_pdbaccession2taxID.split('/')[-1], options.threads) else: accession.read_accessions( database_folder / url_protaccession2taxID.split('/')[-1], database_folder / url_pdbaccession2taxID.split('/')[-1], options.threads) logger.debug('All accession IDs collected.') logger.info('Start writing taxon selected peptide database to %s.' % output_path) wc = WriteCustomDB(path_to_db, output_path) wc.read_database(False, gzipped=TestFile.test_gzipped(path_to_db), accessions=accession.accessionIDs, threads=options.threads) logger.debug('End writing taxon selected peptide database.') # non redundant database # uniprot: write custom specified db to --out else: logger.info('Start writing taxon selected peptide database to %s.' % output_path) wc = WriteCustomDB(path_to_db, output_path, final_taxIDs) wc.read_database(True, threads=options.threads, gzipped=TestFile.test_gzipped(path_to_db)) logger.debug('End writing taxon selected peptide database.') # non redundant database if options.non_redundant: DatabaseCleaner.non_redundant(output_path, with_taxon_ID) # remove redundant database: output_path.unlink() if options.reduce_header and not with_taxon_ID: # reduce headers of NCBI database DatabaseCleaner.reduce_header(output_path) output_path.unlink() logger.info('Program finished.') exit(0)
#!/usr/bin/env python #coding=utf-8 # # Author: Archer Reilly # Date: 11/Aug/2014 # File: PlayerInfoParserTest.py # Description: test the PlayerInfoParser class # Website: http://csrgxtu.blog.com/ # # Produced By CSRGXTU from PlayerInfoParser import PlayerInfoParser from Download import Download URL = "http://sports.qq.com/d/f_players/3/2890/" player = Download(URL) if player.doRequest() != 0: print "Download Cant Do Requst" else: print "Successfully Do Request" playerParser = PlayerInfoParser(player.getSOURCE())
#coding=utf-8 import Path from Download import Download import NameList as NL download = Download() def updateHistory(): print 'Updateing history ...' array = NL.getCodes() download.downHistory(array) print 'Update history success ...' raw_input() if __name__ == '__main__': updateHistory()
def doRequest(self, url): d = Download(url) if d.doRequest() == None: return None else: return d.getSOURCE()
def __init__(self, loop, url): self.__url = self.__remove_trailing_slash(url) self.__loop = loop self.__downloader = Download() self.__scraper = Scraper()
def __init__(self): self.__download = Download()
#!/usr/bin/env python #coding=utf-8 # # Author: Archer Reilly # Date: 05/Aug/2014 # File: DownloadTest.py # Description: test Download class # Website: https://csrgxtu.blog.com # # Produced By CSRGXTU from Download import Download soccer = Download("http://sports.qq.com/d/f_teams/1/42/") if soccer.doRequest() == 0: print "Successfully do request" else: print "Failed do request" html = soccer.getSOURCE() html_unicode = unicode(html, soccer.getEncoding()) html_utf8 = html_unicode.encode("utf8") print html_utf8
def action(): Download()
def stampaRicerca(self): print('Risultati ricerca:') for index in range(0, len(self.listPeers)): print('\n', index + 1, '- descrizione: ', self.listPeers[index][1]) # Formato listPeers: # 0 1 2 3 # 0 1 2 # [ md5 | nome_file | n_copie [ peer_IPv4 | peer_IPv6 | peer_porta ] ] flag = True while flag: flag = False print('\nIndicare quale si desidera scaricare (0 per annullare):') choice = int(input()) if choice == 0: os.system('clear') flag = False elif not choice in range(1, len(self.listPeers) + 1): print('La risorsa non esiste, ritenta') flag = True else: self.index_md5 = choice for copy in range(len(self.listPeers[choice - 1][3])): print('\n', copy + 1, '- \n\tIPv4P2P: \t', self.listPeers[index][3][copy][0], '\n\tIPv6P2P: \t', self.listPeers[index][3][copy][1], '\n\tPP2P: \t\t', self.listPeers[index][3][copy][2]) #Formattazione IPv4 eliminando gli zeri non necessari self.split_ip = self.listPeers[index][3][copy][0].split(".") self.ipp2p = self.split_ip[0].lstrip( '0') + '.' + self.split_ip[1].lstrip( '0') + '.' + self.split_ip[2].lstrip( '0') + '.' + self.split_ip[3].lstrip('0') self.ipp2p_6 = str( ipaddr.ip_address(self.listPeers[index][3][copy][1])) print(self.ipp2p_6) flag = True while flag: flag = False print( 'Indicare da quale peer scaricare il file selezionato (0 per annullare):' ) choicePeer = int(input()) if choicePeer == 0: print('Abortito') flag = False os.system('clear') elif not choicePeer in range( 1, len(self.listPeers[choice - 1][3]) + 1): print('Il peer non esiste, ritenta') flag = True else: print( self.sID, self.ipp2p, self.ipp2p_6, self.listPeers[self.index_md5 - 1][3][choicePeer - 1][2], self.listPeers[self.index_md5 - 1][0], self.listPeers[self.index_md5 - 1][1], self.ipp2p_dir_4, self.ipp2p_dir_6) down = Download( self.sID, self.ipp2p, self.ipp2p_6, self.listPeers[self.index_md5 - 1][3][choicePeer - 1][2], self.listPeers[self.index_md5 - 1][0], self.listPeers[self.index_md5 - 1][1], self.ipp2p_dir_4, self.ipp2p_dir_6) down.download()
def __init__(self, in_dir, exts='.jpg'): # full-path della cartella di input in_dir = os.path.abspath(in_dir) self.in_dir = in_dir # converto tutte le estensioni al carattere minuscolo self.exts = tuple(ext.lower() for ext in exts) # Nomi per le classi self.class_names = [] # tutti i nomi dei file nel training set self.filenames = [] # nomi dei file nel test set self.filenames_test = [] # classe di ogni elemento nel training set self.class_numbers = [] # classe di ogni elemento nel test set self.class_numbers_test = [] # numero totale di classi nel data-set self.num_classes = 0 # istanza per il downlaod self.download = Download() #ritorna tutti i nomi dentro alla cartella for name in os.listdir(in_dir): current_dir = os.path.join(in_dir, name) if os.path.isdir(current_dir): self.class_names.append(name) #training set filenames = self._get_filenames(current_dir) self.filenames.extend(filenames) class_number = self.num_classes class_numbers = [class_number] * len(filenames) self.class_numbers.extend(class_numbers) #test-set filenames_test = self._get_filenames( os.path.join(current_dir, 'test')) self.filenames_test.extend(filenames_test) class_numbers = [class_number] * len(filenames_test) self.class_numbers_test.extend(class_numbers) self.num_classes += 1
# Description: test Parser class # Website: http://csrgxtu.blog.com/ # # Produced By CSRGXTU import requests from Download import Download from Parser import Parser from TeamInfoParser import TeamInfoParser """ page = requests.get('http://econpy.pythonanywhere.com/ex/001.html') print page.text parser = Parser(page.text) #print parser.getBuyers() """ URL = "http://sports.qq.com/d/f_teams/1/42/" soccer = Download(URL) if soccer.doRequest() == 0: print "Successfully do request" else: print "Failed do request" html = soccer.getSOURCE() parser = TeamInfoParser(html) name = parser.getTeamName() print "name:", unicode(name).encode('utf8') name_cn = parser.getTeamNameCN() print "name_cn:", unicode(name_cn).encode('utf8') logo = parser.getTeamLogo() print "logo:", logo city = parser.getTeamCity() print "city:", city
#print("...reading response from other peers...") print("Research termined\nResult for " + pktid) print("[select with the number the file to download]") res = db.retrieveResponses(pktid) if (len(res) == 0): print("File not found") else: choice_list = [] i = 1 for row in res: print( str(i) + ") " + "ip: " + row[1] + " port: " + row[2] + " md5: " + row[3] + " file: " + row[4]) choice_list.append(row) i = i + 1 choice = input(bcolors.OKBLUE + ">> " + bcolors.ENDC) peer = choice_list[int(choice) - 1] addr = Util.ip_deformatting(peer[1], peer[2], None) ip6 = ipad.ip_address(peer[1][16:]) down = Download(str(addr[0]), str(ip6), peer[2], peer[3], peer[4].rstrip()) down.download() print("\n--- FILE DOWNLOADED ---\n")
# copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. from FileCheck import FileCheck from Rename import Rename from Resize import Resize from Download import Download import os Download() FileCheck().verify() FileCheck().remove() Rename() Resize() os.system('python retrain.py ' '--bottleneck_dir=bottlenecks ' '--model_dir=inception ' '--summaries_dir=training_summaries/long ' '--output_graph=retrained_graph.pb ' '--output_labels=retrained_labels.txt ' '--image_dir=imageset/')
example['tags'] = [ 'full color' ] example['start'] = 2 example['len'] = 1 url += '/' + example['prior'] url += '/' + (example[example['prior']][0]).replace(' ', '+') if example['isNew'] !='yes': url += '/' + 'popular?order=' if url.find('?') == -1: url += '?page={}' else: url += '&page={}' taskList = list() for index in range(example['len']): print(url.format(example['start']+index)) bs = BeautifulSoup(requests.get(url.format(example['start']+index)).content,'lxml') taskList.extend([result.find('a')['href'] for result in bs.findAll(class_='gallery')]) for url in taskList: Download(website + url, example).start()