def main(): global startTrail,prevTrail,conn try: startIdx = int(sys.argv[1]) if len(sys.argv)>1 else 0 conn= sqlite3.connect('arris.sqlite3') csr=conn.cursor() rows = csr.execute( "SELECT id,file_url,file_sha1 FROM TFiles ORDER BY id " "LIMIT -1 OFFSET %d"%startIdx ).fetchall() for idx, row in enumerate(rows,startIdx): devId,file_url,file_sha1 = row if not file_url: continue if file_sha1: continue uprint('idx=%d'%idx) try: local_file = downloadFile(file_url, "Content-Disposition") except TypeError: continue file_sha1 = getFileSha1(local_file) file_size = path.getsize(local_file) csr.execute( "UPDATE TFiles SET file_sha1=:file_sha1,file_size=:file_size" " WHERE id = :devId", locals()) conn.commit() ftp = ftputil.FTPHost(ftpHostName,ftpUserName,ftpPassword) uprint('upload to GRID') ftp.upload(local_file, path.basename(local_file)) ftp.close() os.remove(local_file) except Exception as ex: ipdb.set_trace() traceback.print_exc()
def main(): global startTrail,prevTrail,conn try: startIdx = int(sys.argv[1]) if len(sys.argv)>1 else 0 conn= sqlite3.connect('huawei_consumer_search_by_keyword.sqlite3') csr=conn.cursor() rows = csr.execute( "SELECT id,file_name,file_url,file_sha1,file_size FROM TFiles" " ORDER BY id LIMIT -1 OFFSET %d"%startIdx ).fetchall() for idx, row in enumerate(rows,startIdx): devId,file_name,file_url,file_sha1,file_size = row if not file_url: continue if file_sha1: continue if 'Android' in file_name: continue if 'Ascend' in file_name: continue if 'Honor' in file_name: continue if 'Open Source' in file_name: continue if 'opensource' in file_name.lower(): continue uprint('idx=%d, file_name="%s",file_size=%d'%(idx,file_name,file_size)) model = guessModel(file_name) uprint('model="%s"'%model) if not model: continue fw_ver = guessVersion(file_name) uprint('fw_ver="%s"'%fw_ver) if not fw_ver: continue try: local_file = downloadFile(file_url, "Content-Disposition") except TypeError: continue file_sha1 = getFileSha1(local_file) file_size = path.getsize(local_file) csr.execute( "UPDATE TFiles SET file_sha1=:file_sha1" ",file_size=:file_size" ",model=:model " ",fw_ver=:fw_ver " " WHERE id = :devId", locals()) conn.commit() ftp = ftputil.FTPHost(ftpHostName,ftpUserName,ftpPassword) uprint('upload to GRID') ftp.upload(local_file, path.basename(local_file)) ftp.close() os.remove(local_file) except Exception as ex: ipdb.set_trace() traceback.print_exc()
def main(): startIdx = int(sys.argv[1]) if len(sys.argv)>1 else 0 global conn with sqlite3.connect('belkin.sqlite3') as conn: csr=conn.cursor() rows=csr.execute("SELECT id,download_url,page_url,file_sha1" " FROM TFiles LIMIT -1 OFFSET %d"%startIdx).fetchall() for idx, row in enumerate(rows,startIdx): devId,url,page_url,fileSha1=row if fileSha1 or not url: continue uprint('idx=%s, url=%s'%(idx,url)) url = safeUrl(url) uprint('url='+url) fname=urlFileName(url) uprint('download "%s"'%(fname)) fname = path.join(dlDir, fname) try: downloadFile(url, fname) except urllib.error.HTTPError: print(ex) continue except OSError as ex: if ex.errno == 28: print(ex);print('[Errno 28] No space left on device') break except Exception as ex: ipdb.set_trace() traceback.print_exc() continue fileSha1=getFileSha1(fname) fileSize=path.getsize(fname) print('sha1="%s" for "%s"'%(fileSha1,fname)) csr.execute("UPDATE TFiles SET file_sha1=:fileSha1," " file_size=:fileSize WHERE id=:devId", locals()) conn.commit() uprint('UPDATE fileSha1=%(fileSha1)s, fileSize=%(fileSize)s' ' WHERE id="%(devId)s"' %locals()) with ftputil.FTPHost(ftpHostName,ftpUserName,ftpPassword) as ftp: ftp.upload(fname, path.basename(fname)) uprint('uploaded "%s" to ftp://%s/' %(path.basename(fname), ftpHostName)) os.remove(fname)
def main(): startIdx = int(sys.argv[1]) if len(sys.argv) > 1 else 0 global conn with sqlite3.connect("Linksys.sqlite3") as conn: csr = conn.cursor() rows = csr.execute( "SELECT brand,model,revision,file_title,href,file_sha1 FROM TFiles " "LIMIT -1 OFFSET %d" % startIdx ).fetchall() for idx, row in enumerate(rows, startIdx): brand, model, revision, fileTitle, url, fileSha1 = row print("idx= %d, fileSha1=%s" % (idx, fileSha1)) if fileSha1: uprint('"%s" already downloaded, bypass!' % fileTitle) continue if not url: continue fname = urlFileName(url) if not fname: fname = safeFileName(fileTitle) uprint("url=" + url) uprint('download "%s" as "%s"' % (fileTitle, fname)) fname = path.join(dlDir, fname) try: downloadFile(url, fname) except urllib.error.HTTPError: print(ex) continue except OSError as ex: if ex.errno == 28: print(ex) print("[Errno 28] No space left on device") break except Exception as ex: import pdb pdb.set_trace() import traceback traceback.print_exc() print(ex) continue fileSha1 = getFileSha1(fname) fileSize = path.getsize(fname) print('sha1="%s" for "%s"' % (fileSha1, fname)) csr.execute( "UPDATE TFiles SET file_sha1=:fileSha1, file_size=:fileSize" " WHERE brand=:brand AND model=:model AND revision=:revision" " AND file_title=:fileTitle", locals(), ) conn.commit() print( "UPDATE fileSha1=%(fileSha1)s, fileSize=%(fileSize)d" ' WHERE "%(brand)s", "%(model)s", "%(revision)s", ' '"%(fileTitle)s" ' % locals() ) with ftputil.FTPHost(ftpHostName, ftpUserName, ftpPassword) as ftp: ftp.upload(fname, path.basename(fname)) print('uploaded "%s" to ftp://%s/' % (path.basename(fname), ftpHostName)) os.remove(fname)