def golive(config, shelvedb_paths=[]):
if shelvedb_paths:
csvfilepath = None
for shelvedb_path in shelvedb_paths:
columns = []
name = shelvedb_path.rpartition('/')[-1]
for dimension in config.dimensions:
if name == dimension.name:
columns = dimension.all
break
shelveddb = LRUShelve(shelvedb_path, 2000, readonly=True)
fd, csvfilepath = tempfile.mkstemp(suffix='.csv', prefix=name)
tmpfile = file(csvfilepath, 'w')
csvwriter = UnicodeWriter(tmpfile, delimiter='\t')
for key, rows in shelveddb.iterms():
for row in rows:
values = []
for i in range(len(columns)):
value = row[i]
if value == None:
value = ''
values.append(value)
csvwriter.writerow(values)
tmpfile.close()
os.close(fd)
config.UDF_pgcopy(name, columns, '\t', None, '', file(csvfilepath))
shelveddb.close()
os.remove(csvfilepath)
def write_csv(info_dics, output):
"""
write infomation to csv
"""
with open(output, 'w+') as o:
fw = UnicodeWriter(o, encoding="utf-8")
fw.writerow(info_dics[0].keys())
for d in info_dics:
fw.writerow(d.values())
def _combine_csv(self, output):
with open(output, 'w+') as o:
fw = UnicodeWriter(o)
fw.writerow(table_title)
for file in self.files:
if not file.endswith('.csv'):
continue
itable = self._read_restruct_csv(file)
for i in itable:
fw.writerow(i)
def shelved2onlinedb(self):
if len(self.shelveddb) > 0:
csvfilepath = None
curs = None
try:
fd, csvfilepath = tempfile.mkstemp(suffix='.csv',
prefix=self.name)
tmpfile = file(csvfilepath, 'w')
csvwriter = UnicodeWriter(tmpfile, delimiter='\t')
for k in self.shelveddb:
csvwriter.writerows(self.shelveddb[k])
tmpfile.close()
os.close(fd)
curs = self.con.cursor()
curs.execute('TRUNCATE %s' % self.name)
curs.copy_from(file=file(csvfilepath),
table=self.name,
sep='\t',
null='',
columns=self.all)
finally:
if curs:
curs.close()
os.remove(csvfilepath)
try:
label = tag[u'android:label']
except KeyError:
label = app_label
if label.startswith(u'@ref/'):
label = get_label(label[len(u'@ref/'):], pkg, apk)
name = tag[u'android:name']
if name.startswith('.'):
name = pkg + name
star = apk.split('/')[-1][:-len('.apk')]
print('[%4d] %s %s "%s"' % (i, star, name, label))
apps.append([star, name, label])
for apk in glob.glob('../apks/wear/*.apk'):
i += 1
run(i, apk)
with open('pkg_wfs_label.csv', 'wb') as f:
writer = UnicodeWriter(f)
# writer = csv.writer(f, encoding='utf-8')
for wf in apps:
print('[write] %s' % wf)
writer.writerow(wf)
with open('pkg_wfs_label.csv', 'rb') as f:
reader = UnicodeReader(f)
# reader = (f, encoding='utf-8')
for row in reader:
print('[read] %s' % row)
fs=fields.split("|")
return set(fs)==set(filter(lambda a: h[a],h.keys()))
def output_row(h,row) :
rrow=[]
for f in row :
if f in h:
rrow.append(h[f])
else :
rrow.append("")
return rrow
ofile=UnicodeWriter(open(files["csv"],"w"))
for fn in files["in"] :
try :
tree=etree.parse(open(fn),etree.HTMLParser())
data=[]
drows=0
props=fnmeta(fn)
props["file"]=os.path.split(fn)[1]
for (k,v) in parse["properties"].items() :
a=stringstring(extract(tree,v))
if a is not None:
props[k]=a
#pprint.pprint(props)
for container in tree.xpath(parse["container"]) :
cdata=[]
res.append(r)
return res
rr=[]
for f in input_files.split("\n")[:5] :
try :
n=parsefile(f)
if len(n)>0 :
rr.extend(n)
print "%s - %s" % (len(n),f)
else :
print "!! - %s" % (f,)
except Exception, e:
raise
print "!!! - %s - %s" % (f,e)
import simplejson,string
rr_germany=filter(lambda a : string.lower("%(category)s %(share)s" % a).find("germany")>-1,rr)
simplejson.dump(rr,open(os.path.join(output_dir,"resultate-runde2.json"),"w"))
simplejson.dump(rr_germany,open(os.path.join(output_dir,"resultate-runde2-germany.json"),"w"))
from unicodecsv import UnicodeWriter
uw=UnicodeWriter(open(os.path.join(output_dir,"resultate-runde2.csv"),"w"),delimiter=";")
uw.writerows(map(lambda a: [a["date"],a["category"],a.get("fund",""),a["share"],a["number"],a["price"],a["file"]], rr))
uw=UnicodeWriter(open(os.path.join(output_dir,"resultate-runde2-germany.csv"),"w"),delimiter=";")
uw.writerows(map(lambda a: [a["date"],a["category"],a.get("fund",""),a["share"],a["number"],a["price"],a["file"]], rr_germany))
res.append(r)
return res
rr=[]
for f in ffn.split("\n") :
try :
n=parsefile(f)
if len(n)>0 :
rr.extend(n)
print "%s - %s" % (len(n),f)
else :
print "!! - %s" % (f,)
except Exception, e:
raise
print "!!! - %s - %s" % (f,e)
import simplejson,string
rr_germany=filter(lambda a : string.lower("%(category)s %(share)s" % a).find("germany")>-1,rr)
simplejson.dump(rr,open("data/resultate-runde2.json","w"))
simplejson.dump(rr_germany,open("resultate-runde2-germany.json","w"))
from unicodecsv import UnicodeWriter
uw=UnicodeWriter(open("data/resultate-runde2.csv","w"),delimiter=";")
uw.writerows(map(lambda a: [a["date"],a["category"],a["share"],a["number"],a["price"],a["file"]], rr))
uw=UnicodeWriter(open("data/resultate-runde2-germany.csv","w"),delimiter=";")
uw.writerows(map(lambda a: [a["date"],a["category"],a["share"],a["number"],a["price"],a["file"]], rr_germany))
return res
rr=[]
for f in input_files:
try :
n=parsefile(f)
if n :
print "%s - %s" % (len(n),f)
rr.extend(n)
else :
print "!! - %s " % (f,)
except Exception, e:
raise
print "!!! - %s - %s" % (f,e)
_me=os.path.splitext(os.path.split(f)[1])[0]
import simplejson,string
rr_germany=filter(lambda a : string.lower("%(category)s %(share)s %(fund)s" % defaultdict(lambda:"",a)).find("germany")>-1,rr)
simplejson.dump(rr,open(os.path.join(output_dir,"resultate-%s.json" % _me ),"w"))
simplejson.dump(rr_germany,open(os.path.join(output_dir,"resultate-%s-germany.json" % _me),"w"))
from unicodecsv import UnicodeWriter
uw=UnicodeWriter(open(os.path.join(output_dir,"resultate-%s.csv" % _me),"w"),delimiter=";")
uw.writerows(map(lambda a: [a["date"],a["category"],a.get("fund",""),a["share"],a["number"],a["price"],a["file"]], rr))
uw=UnicodeWriter(open(os.path.join(output_dir,"resultate-%s-germany.csv" % _me),"w"),delimiter=";")
uw.writerows(map(lambda a: [a["date"],a["category"],a.get("fund",""),a["share"],a["number"],a["price"],a["file"]], rr_germany))
def bucket_download(request, id):
# TODO: access control
message = ""
bucket = get_bucket(request, id)
if request.method == 'POST':
try:
zipfile_name = request.POST['zipfile_name']
except KeyError:
zipfile_name = bucket.default_zipfile_name()
if zipfile_name == "":
messages.error(request, 'No zip file name selected.')
c = RequestContext(request, {'title': 'FeedDB Explorer'})
return render_to_response('explorer/base.html', c)
if not zipfile_name.endswith(".zip"):
zipfile_name += ".zip"
download_choice = request.POST['download_choice']
channel_choice = request.POST['channel_choice']
#meta_option= request.POST['meta_option']
quotechar_char = '"'
#delimiter= request.POST['delimiter']
#if delimiter =="tab":
# delimiter_char = '\t'
#elif delimiter =="comma":
# delimiter_char = ','
#else:
delimiter_char = ','
#get selected fields
field_selected = []
for item in request.POST.items():
if (item[1] == "on" and item[0].startswith("chk:")):
field_selected.append(item[0])
message += item[0] + "\n"
if (download_choice == "0"
or download_choice == "2") and len(field_selected) == 0:
messages.error(request, 'No fields selected.')
c = RequestContext(request, {'title': 'FeedDB Explorer'})
return render_to_response('explorer/base.html', c)
meta_selected = {}
for field in field_selected:
parts = field.split(":")
if not parts[1] in meta_selected:
meta_selected[parts[1]] = []
parameter = parts[1] + ":" + parts[2]
meta_selected[parts[1]].append([parts[2], request.POST[parameter]])
#get selected channels
channel_selected = []
channel_headers = []
for item in request.POST.items():
if (item[1] == "on" and item[0].startswith("channel:")):
channel_selected.append(item[0])
message += item[0] + "\n"
if (channel_choice == "1" and len(channel_selected) == 0):
messages.error(request, 'No channels selected.')
c = RequestContext(request, {'title': 'FeedDB Explorer'})
return render_to_response('explorer/base.html', c)
channel_download = []
channel_selected.sort()
trials_download = []
for ch in channel_selected:
parts = ch.split(":")
channel_download.append([parts[1], parts[2]])
channel_headers.append("Trial %s:Channel %s" %
(parts[1], parts[2]))
if not parts[1] in trials_download:
trials_download.append(parts[1])
filenames = {}
# create a temporary folder to store files
from time import time
tempdir = settings.EXPLORER_TEMPORARY_FOLDER + "/" + str(
time()).replace('.', '')
try:
os.makedirs(tempdir)
except OSError, err:
messages.error(
request,
'Failed to create folder for storing downloaded files.')
c = RequestContext(request, {'title': 'FeedDB Explorer'})
return render_to_response('explorer/base.html', c)
#
# create meta data if the user has chosen to do so
#
if (download_choice == "0" or download_choice == "2"):
#create trials mate data file and out it into the temp zip file
full_filename = "%s/trials.csv" % tempdir
filenames["trials.csv"] = full_filename
metaWriter = UnicodeWriter(open(full_filename, "w"),
delimiter=delimiter_char,
doublequote='false',
escapechar='\\',
quotechar=quotechar_char,
quoting=csv.QUOTE_MINIMAL)
#output trials
#output headers
headers = ["Trial:ID"]
for key, value in meta_selected.items():
if not key in ('Setup', 'EmgSetup', 'SonoSetup', 'Sensor',
'EmgSensor', 'SonoSensor', 'Channel',
'EmgChannel', 'SonoChannel', 'PressureChannel',
'ForceChannel', 'StrainChannel',
'KinematicsChannel', 'EventChannel'):
for v in value:
headers.append(v[1])
metaWriter.writerow(headers)
objects = {}
for trial in bucket.trials.all():
values = [trial.id]
objects["Session"] = trial.session
objects["Experiment"] = trial.session.experiment
objects["Study"] = trial.session.experiment.study
objects["Subject"] = trial.session.experiment.subject
objects["Trial"] = trial
for key, value in meta_selected.items():
if key in objects:
for v in value:
s = getattr(objects[key], v[0])
if hasattr(s, 'split'):
ss = s.split('\r\n')
if len(ss) > 1:
s = ' '.join(ss)
values.append(s)
metaWriter.writerow(values)
#output channels
#generate channel headers
headers = ["Channel:ID"]
for key, value in meta_selected.items():
#generate headers meta data
if key in ('Setup', 'EmgSetup', 'SonoSetup', 'Sensor',
'EmgSensor', 'SonoSensor', 'Channel', 'EmgChannel',
'SonoChannel', 'PressureChannel', 'ForceChannel',
'StrainChannel', 'KinematicsChannel',
'EventChannel'):
for v in value:
headers.append(v[1])
for key, value in meta_selected.items():
#generate headers for 2 meta data (specifically for crystal2 in sono data
if key in ('Sensor', 'SonoSensor'):
for v in value:
headers.append('Sensor 2:%s' % v[1])
channel_types = [
'strainchannel', 'forcechannel', 'pressurechannel',
'kinematicschannel'
]
for trial in bucket.trials.all():
#trial_name = trial.title.replace('.', '').replace(',', '').replace(' ', '_').strip().lower()
#filename = "trial_%d_%s_channels.csv" % (trial.id, trial_name)
#full_filename = "%s/trial_%d_%s_channels.csv" % (tempdir, trial.id,trial_name)
filename = "trial_%d_channels.csv" % trial.id
full_filename = "%s/trial_%d_channels.csv" % (tempdir,
trial.id)
filenames[filename] = full_filename
f = open(full_filename, "w")
metaWriter = UnicodeWriter(f,
delimiter=delimiter_char,
doublequote='false',
escapechar='\\',
quotechar=quotechar_char,
quoting=csv.QUOTE_MINIMAL)
metaWriter.writerow(headers)
objects = {}
for lineup in trial.session.channellineup_set.all():
objects = {}
ch = lineup.channel
if ch == None:
values = ["deadchannel"]
else:
objects["Channel"] = lineup.channel
values = [ch.id]
objects["Setup"] = ch.setup
for channel_type in channel_types:
if hasattr(ch, channel_type):
objects["Sensor"] = getattr(
ch, channel_type).sensor
if hasattr(ch.setup, 'emgsetup'):
objects["EmgSetup"] = ch.setup.emgsetup
if hasattr(ch.setup, 'sonosetup'):
objects["SonoSetup"] = ch.setup.sonosetup
if hasattr(ch, 'emgchannel'):
objects["EmgChannel"] = ch.emgchannel
objects["Sensor"] = ch.emgchannel.sensor
objects["EmgSensor"] = ch.emgchannel.sensor
if hasattr(ch, 'eventchannel'):
objects["EventChannel"] = ch.eventchannel
if hasattr(ch, 'pressurechannel'):
objects["PressureChannel"] = ch.pressurechannel
if hasattr(ch, 'strainchannel'):
objects["StrainChannel"] = ch.strainchannel
if hasattr(ch, 'forcechannel'):
objects["ForceChannel"] = ch.forcechannel
if hasattr(ch, 'kinematicschannel'):
objects["KinematicsChannel"] = ch.kinematicschannel
if hasattr(ch, 'sonochannel'):
objects["SonoChannel"] = ch.sonochannel
objects["Sensor"] = ch.sonochannel.crystal1
objects["SonoSensor"] = ch.sonochannel.crystal1
if hasattr(ch, 'emgchannel'):
objects["Sensor"] = ch.emgchannel.sensor
for key, value in meta_selected.items():
if key in ('Setup', 'EmgSetup', 'SonoSetup', 'Sensor',
'EmgSensor', 'SonoSensor', 'Channel',
'EmgChannel', 'SonoChannel',
'PressureChannel', 'ForceChannel',
'StrainChannel', 'KinematicsChannel',
'EventChannel'):
for v in value:
s = ''
if key in objects and objects[key] != None:
s = getattr(objects[key], v[0])
if hasattr(
s,
'split'): #check if s is a string
ss = s.split('\r\n')
if len(ss) > 1:
s = ' '.join(ss)
values.append(s)
#output the second crystal sensor information if it is sono channel
if hasattr(ch, 'sonochannel'):
objects["Sensor"] = ch.sonochannel.crystal2
objects["SonoSensor"] = ch.sonochannel.crystal2
for key, value in meta_selected.items():
if key in ('Sensor', 'SonoSensor'):
for v in value:
s = ''
if key in objects:
s = getattr(objects[key], v[0])
if hasattr(s, 'split'):
ss = s.split('\r\n')
if len(ss) > 1:
s = ' '.join(ss)
values.append(s)
metaWriter.writerow(values)
f.close()
#
# put data files into the tmp zip
#
data_files = {}
if (download_choice == "1" or download_choice == "2"):
# download all trial files
if channel_choice == "0":
for trial in bucket.trials.all():
#check if there is a data file
if (trial.data_file != None and trial.data_file != ""):
filename = "trial_%d.dat" % trial.id
full_filename = "%s/%s" % (settings.MEDIA_ROOT,
trial.data_file)
data_files[filename] = full_filename
else:
# download selected channels
filename = "channels.dat"
full_filename = "%s/channels.dat" % tempdir
filenames[filename] = full_filename
f = open(full_filename, "w")
metaWriter = UnicodeWriter(f,
delimiter=delimiter_char,
doublequote='false',
escapechar='\\',
quotechar=quotechar_char,
quoting=csv.QUOTE_MINIMAL)
metaWriter.writerow(channel_headers)
trial_readers = {}
total_trial_number = 0
for trial in bucket.trials.all():
#check if there is a data file
if (trial.data_file != None and trial.data_file != ""
and str(trial.id) in trials_download):
full_filename = "%s/%s" % (settings.MEDIA_ROOT,
trial.data_file)
csvfile = open(full_filename, "rU")
dialect = csv.Sniffer().sniff(csvfile.read(1024))
csvfile.seek(0)
reader = csv.reader(csvfile, dialect)
trial_readers[str(trial.id)] = {
"reader": reader,
"hasmore": True,
"file": csvfile
}
total_trial_number += 1
rows = {}
newrow = []
finished_file_number = 0
while finished_file_number < total_trial_number:
rows.clear()
for key in trial_readers:
try:
if trial_readers[key]["hasmore"]:
row = trial_readers[key]["reader"].next()
rows[key] = row
except StopIteration:
finished_file_number += 1
trial_readers[key]["hasmore"] = False
trial_readers[key]["file"].close()
newrow = []
for ch in channel_download:
if ch[0] in rows:
if int(ch[1]) > len(rows[ch[0]]):
messages.error(
request,
"Error in channel lineup positions for trial: %s"
% ch[0])
c = RequestContext(
request, {'title': 'FeedDB Explorer'})
return render_to_response(
'explorer/base.html', c)
newrow.append(rows[ch[0]][int(ch[1]) - 1])
else:
newrow.append('')
metaWriter.writerow(newrow)
f.close()
response = send_zipfile(request, filenames, data_files, zipfile_name)
for file, full_file in filenames.items():
os.remove(full_file)
os.rmdir(tempdir)
return response
title_words = set([stemmer.stemWord(word.lower())
for word in title.split()
if word not in stopwords
])
if name_words == title_words:
wiki_match, confidence = wikipedia_match(ann)
except Exception as e:
import pdb
pdb.set_trace()
if wiki_match:
outfile_name = new_annotations
if close_match:
close_match_obj = close_match.split('/')[-1]
wiki_match_obj = wiki_match.split('/')[-1]
if close_match_obj != wiki_match_obj:
outfile_name = different_annotations
else:
print name, 'già matchato con: ', close_match
continue
with open(outfile_name, 'a+') as outfile:
writer = UnicodeWriter(outfile)
writer.writerow([subject_url,
name,
wiki_match,
unicode(confidence)
])
