def main():
global startTrail,prevTrail,conn
try:
startIdx = int(sys.argv[1]) if len(sys.argv)>1 else 0
conn= sqlite3.connect('arris.sqlite3')
csr=conn.cursor()
rows = csr.execute(
"SELECT id,file_url,file_sha1 FROM TFiles ORDER BY id "
"LIMIT -1 OFFSET %d"%startIdx
).fetchall()
for idx, row in enumerate(rows,startIdx):
devId,file_url,file_sha1 = row
if not file_url:
continue
if file_sha1:
continue
uprint('idx=%d'%idx)
try:
local_file = downloadFile(file_url, "Content-Disposition")
except TypeError:
continue
file_sha1 = getFileSha1(local_file)
file_size = path.getsize(local_file)
csr.execute(
"UPDATE TFiles SET file_sha1=:file_sha1,file_size=:file_size"
" WHERE id = :devId", locals())
conn.commit()
ftp = ftputil.FTPHost(ftpHostName,ftpUserName,ftpPassword)
uprint('upload to GRID')
ftp.upload(local_file, path.basename(local_file))
ftp.close()
os.remove(local_file)
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
def main():
global startTrail,prevTrail,conn
try:
startIdx = int(sys.argv[1]) if len(sys.argv)>1 else 0
conn= sqlite3.connect('huawei_consumer_search_by_keyword.sqlite3')
csr=conn.cursor()
rows = csr.execute(
"SELECT id,file_name,file_url,file_sha1,file_size FROM TFiles"
" ORDER BY id LIMIT -1 OFFSET %d"%startIdx
).fetchall()
for idx, row in enumerate(rows,startIdx):
devId,file_name,file_url,file_sha1,file_size = row
if not file_url:
continue
if file_sha1:
continue
if 'Android' in file_name:
continue
if 'Ascend' in file_name:
continue
if 'Honor' in file_name:
continue
if 'Open Source' in file_name:
continue
if 'opensource' in file_name.lower():
continue
uprint('idx=%d, file_name="%s",file_size=%d'%(idx,file_name,file_size))
model = guessModel(file_name)
uprint('model="%s"'%model)
if not model:
continue
fw_ver = guessVersion(file_name)
uprint('fw_ver="%s"'%fw_ver)
if not fw_ver:
continue
try:
local_file = downloadFile(file_url, "Content-Disposition")
except TypeError:
continue
file_sha1 = getFileSha1(local_file)
file_size = path.getsize(local_file)
csr.execute(
"UPDATE TFiles SET file_sha1=:file_sha1"
",file_size=:file_size"
",model=:model "
",fw_ver=:fw_ver "
" WHERE id = :devId", locals())
conn.commit()
ftp = ftputil.FTPHost(ftpHostName,ftpUserName,ftpPassword)
uprint('upload to GRID')
ftp.upload(local_file, path.basename(local_file))
ftp.close()
os.remove(local_file)
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
def main():
startIdx = int(sys.argv[1]) if len(sys.argv)>1 else 0
global conn
with sqlite3.connect('belkin.sqlite3') as conn:
csr=conn.cursor()
rows=csr.execute("SELECT id,download_url,page_url,file_sha1"
" FROM TFiles LIMIT -1 OFFSET %d"%startIdx).fetchall()
for idx, row in enumerate(rows,startIdx):
devId,url,page_url,fileSha1=row
if fileSha1 or not url:
continue
uprint('idx=%s, url=%s'%(idx,url))
url = safeUrl(url)
uprint('url='+url)
fname=urlFileName(url)
uprint('download "%s"'%(fname))
fname = path.join(dlDir, fname)
try:
downloadFile(url, fname)
except urllib.error.HTTPError:
print(ex)
continue
except OSError as ex:
if ex.errno == 28:
print(ex);print('[Errno 28] No space left on device')
break
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
continue
fileSha1=getFileSha1(fname)
fileSize=path.getsize(fname)
print('sha1="%s" for "%s"'%(fileSha1,fname))
csr.execute("UPDATE TFiles SET file_sha1=:fileSha1,"
" file_size=:fileSize WHERE id=:devId", locals())
conn.commit()
uprint('UPDATE fileSha1=%(fileSha1)s, fileSize=%(fileSize)s'
' WHERE id="%(devId)s"' %locals())
with ftputil.FTPHost(ftpHostName,ftpUserName,ftpPassword) as ftp:
ftp.upload(fname, path.basename(fname))
uprint('uploaded "%s" to ftp://%s/'
%(path.basename(fname), ftpHostName))
os.remove(fname)
def main():
startIdx = int(sys.argv[1]) if len(sys.argv) > 1 else 0
global conn
with sqlite3.connect("Linksys.sqlite3") as conn:
csr = conn.cursor()
rows = csr.execute(
"SELECT brand,model,revision,file_title,href,file_sha1 FROM TFiles " "LIMIT -1 OFFSET %d" % startIdx
).fetchall()
for idx, row in enumerate(rows, startIdx):
brand, model, revision, fileTitle, url, fileSha1 = row
print("idx= %d, fileSha1=%s" % (idx, fileSha1))
if fileSha1:
uprint('"%s" already downloaded, bypass!' % fileTitle)
continue
if not url:
continue
fname = urlFileName(url)
if not fname:
fname = safeFileName(fileTitle)
uprint("url=" + url)
uprint('download "%s" as "%s"' % (fileTitle, fname))
fname = path.join(dlDir, fname)
try:
downloadFile(url, fname)
except urllib.error.HTTPError:
print(ex)
continue
except OSError as ex:
if ex.errno == 28:
print(ex)
print("[Errno 28] No space left on device")
break
except Exception as ex:
import pdb
pdb.set_trace()
import traceback
traceback.print_exc()
print(ex)
continue
fileSha1 = getFileSha1(fname)
fileSize = path.getsize(fname)
print('sha1="%s" for "%s"' % (fileSha1, fname))
csr.execute(
"UPDATE TFiles SET file_sha1=:fileSha1, file_size=:fileSize"
" WHERE brand=:brand AND model=:model AND revision=:revision"
" AND file_title=:fileTitle",
locals(),
)
conn.commit()
print(
"UPDATE fileSha1=%(fileSha1)s, fileSize=%(fileSize)d"
' WHERE "%(brand)s", "%(model)s", "%(revision)s", '
'"%(fileTitle)s" ' % locals()
)
with ftputil.FTPHost(ftpHostName, ftpUserName, ftpPassword) as ftp:
ftp.upload(fname, path.basename(fname))
print('uploaded "%s" to ftp://%s/' % (path.basename(fname), ftpHostName))
os.remove(fname)