forked from rawler/bhindex
/
add.py
executable file
·172 lines (143 loc) · 6.38 KB
/
add.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os, os.path as path, sys
from ConfigParser import NoOptionError
from time import time
import subprocess
HERE = path.dirname(__file__)
sys.path.append(HERE)
import db, config, export_txt, export_links, magnet, scraper
config = config.read()
try:
bh_bindir = config.get('BITHORDE', 'bindir')
bh_bindir = path.expanduser(bh_bindir)
os.environ['PATH'] = "%s:%s" % (bh_bindir, os.environ['PATH'])
except NoOptionError:
pass
bh_upload_bin = 'bhupload'
def sanitizedpath(file):
'''Assumes path is normalized through path.normpath first,
then santizies by removing leading path-fixtures such as [~./]'''
return file.lstrip('./~') # Remove .. and similar placeholders
def bh_upload(file, link):
'''Assumes file is normalized through path.normpath'''
cmd = [bh_upload_bin]
if link:
cmd.append('--link')
cmd.append(file)
bhup = subprocess.Popen(cmd, stdout=subprocess.PIPE)
bhup_out, _ = bhup.communicate()
for line in bhup_out.splitlines():
try:
proto, _ = line.split(':', 1)
except ValueError:
continue
if proto == 'magnet':
return magnet.objectFromMagnet(line.strip().decode('utf8'), fullPath=unicode(file))
return None
if __name__ == '__main__':
import cliopt
usage = "usage: %prog [options] file1/dir1 [file2/dir2 ...]\n" \
" An argument of '-' will expand to filenames read line for line on standard input."
parser = cliopt.OptionParser(usage=usage)
parser.add_option("-l", "--upload_link", action="store_true", dest="upload_link",
default=config.getboolean('BITHORDE', 'upload_link'),
help="Upload as a linked asset, instead of adding it to cache-dir. NEEDS appropriate bithorded-config.")
parser.add_option("-L", "--no-links", action="store_false",
dest="export_links", default=True,
help="When done, don't immediately export links to links-directory")
parser.add_option("-T", "--no-txt", action="store_false",
dest="export_txt", default=True,
help="When done, don't immediately publish txt-format index")
parser.add_option("-t", "--tag", action="append", dest="tags",
help="Define a tag for these uploads, such as '-tname:monkey'")
parser.add_option("-s", "--strip-path", action="store_const", dest="sanitizer",
default=sanitizedpath, const=path.basename,
help="Strip name to just the name of the file, without path")
parser.add_option("-f", "--force",
action="store_true", dest="force", default=False,
help="Force upload even of assets already found in sync in index")
parser.add_option("-r", "--recursive-add",
action="store_true", dest="recursive", default=False,
help="Recursively add files from a folder")
parser.add_option("-e", "--ext",
action="append", dest="exts",
help="Define file extensions to be added. Only valid with -r / --recursive-add")
(options, args) = parser.parse_args()
if len(args) < 1:
parser.error("At least one file or directory must be specified.")
do_export = False
# Parse into DB-tag-objects
tags = cliopt.parse_attrs(options.tags)
DB = db.open(config)
def add(file, tags, exts):
'''Try to upload one file to bithorde and add to index'''
file = unicode(path.normpath(file), sys.stdin.encoding or 'UTF-8')
name = options.sanitizer(file)
mtime = path.getmtime(file)
ext = os.path.splitext(file)[1]
tags = tags or {}
exts = exts or set()
if type(exts).__name__ == 'list':
exts = set(exts)
try:
for e in config.get('ADD', 'extensions').split(','):
exts.add(e.strip())
except NoOptionError:
pass
if ext.strip('.') not in exts and options.recursive == True:
print "* Skipping %s because of extension %s (Add extensions to be included with -e)." % (name, ext)
return
if not options.force:
oldassets = DB.query({'path': name})
found_up2date = False
for a in oldassets:
if a['name'].t >= mtime:
found_up2date = True
if found_up2date:
print "* File \"%s\" already in database, won't add." % file
return
asset = bh_upload(file, options.upload_link)
if asset:
asset.name = name
t = time()
for k,v in tags.iteritems():
v.t = t
asset[k] = v
scraper.scrape_for(asset)
for k,v in sorted(asset.iteritems()):
print (u"%s: %s" % (k, v.join())).encode('utf-8')
DB.update(asset)
global do_export
do_export = True
else:
print "Error adding %s" % file
try:
for arg in args:
if arg == '-':
for line in sys.stdin:
add(line.strip(), options.tags)
else:
if os.path.isdir(arg):
if options.recursive == True:
fileList = []
for root, subFolders, files in os.walk(arg):
subFolders[:] = [d for d in subFolders if not d.startswith('.')] # Remove dotted dirs.
for file in files:
fileList.append(os.path.join(root, file))
for file in fileList:
print "* Considering %s" % file
add(file, options.tags, options.exts)
print "* Processed %s files." % len(fileList)
else:
print "%s is a directory. Perhaps you'd like to add all files from it by specifying -r or --recursive-add?" % arg
elif os.path.isfile(arg):
add(arg, options.tags, options.exts)
else:
parser.error("At least one file or directory must be specified.")
finally:
DB.commit()
if options.export_links and do_export == True:
export_links.main()
if options.export_txt and do_export == True:
export_txt.main()