def getcheck_id(json, link):
id = json.split('dPageId')[1].split('\': ')[1].split(',')[0]
status = json.split('status')[1].split('\': u\'')[1].split('\',')[0]
# happens on some videos
if (status == 'failed'):
print('[!!] Failed to convert link!')
return -1
# filter 'u' from id
# not found
if (id.find('u') == -1):
print('Id: ' + id + '\n')
if (id == '0'):
send_payload_ok(json, link)
else:
send_payload_default(id)
# found
elif (id.find('u') == 0):
id = id.split('u')[1].split('\'')[1]
print('[i] Id: ' + id)
if (id == '0'):
send_payload_ok(json, link)
else:
send_payload_default(id)
else:
print('[!!] Something bad happened!')
return -1
def main():
jsons = glob.glob(BasePath + "numbered/jsons/*.json")
images = glob.glob(BasePath + "numbered/images/*.jpg")
for json in tqdm(jsons):
if len(read_json(json)["bboxes"]) == 0:
shutil.move(json, BasePath + "numbered/jsonmove/{}".format(json.split("/")[-1]))
for image in images:
if image.split("/")[-1].split(".")[0] == json.split("/")[-1].split(".")[0]:
shutil.move(image, BasePath + "numbered/imagemove/{}".format(image.split("/")[-1]))
def saveDataToExcel(json):
datas = read_json(json)
maxnum = 0
for rowdata in datas:
if len(rowdata) > maxnum:
maxnum = len(rowdata)
formdata = {}
for i in range(maxnum):
formdata["{}".format(i)] = []
print(len(formdata))
for rowdata in datas:
i = 0
for j in range(len(formdata)):
# print(j)
if i < len(rowdata):
# print(len(rowdata), i)
formdata[str(j)].append(rowdata[i]["className"])
else:
formdata[str(j)].append('')
i += 1
writer = pd.ExcelWriter('lmrecognnition.xlsx')
df1 = pd.DataFrame(data=formdata)
df1.to_excel(writer,
sheet_name='{}'.format(json.split("/")[-1].split(".")[0]))
writer.save()
def test_metadata3_exceptions():
with pytest.raises(KeyError):
# dict must have a key named 'type'
Metadata3.decode_dtype({})
required = [
"zarr_format", "metadata_encoding", "metadata_key_suffix", "extensions"
]
for key in required:
meta = copy.copy(_default_entry_point_metadata_v3)
meta.pop('zarr_format')
with pytest.raises(ValueError):
# cannot encode metadata that is missing a required key
Metadata3.encode_hierarchy_metadata(meta)
meta = copy.copy(_default_entry_point_metadata_v3)
meta["extra_key"] = []
with pytest.raises(ValueError):
# cannot encode metadata that has unexpected keys
Metadata3.encode_hierarchy_metadata(meta)
json = Metadata3.encode_hierarchy_metadata(
_default_entry_point_metadata_v3)
# form a new json bytes object with one of the keys missing
temp = json.split(b'\n')
temp = temp[:2] + temp[3:]
bad_json = b'\n'.join(temp)
with pytest.raises(ValueError):
# cannot encode metadata that is missing a required key
Metadata3.decode_hierarchy_metadata(bad_json)
json = Metadata3.encode_hierarchy_metadata(
_default_entry_point_metadata_v3)
temp = json.split(b'\n')
temp = temp[:2] + [b' "unexpected": [],'] + temp[2:]
bad_json = b'\n'.join(temp)
with pytest.raises(ValueError):
# cannot encode metadata that has extra, unexpected keys
Metadata3.decode_hierarchy_metadata(bad_json)
codec_meta = dict(configuration=None, codec='unknown')
with pytest.raises(NotImplementedError):
Metadata3._decode_codec_metadata(codec_meta)
with pytest.raises(MetadataError):
Metadata3.decode_array_metadata(dict())
def __StripComments(json: str) -> str:
lines = json.split("\n")
finalLines = []
for line in lines:
strippedLine = line.strip()
if not strippedLine.startswith("//"):
finalLines.append(line)
return "\n".join(finalLines)
def test_dump_json():
# GIVEN some dict
data = {'name': 'PT Anderson', 'age': 45}
# WHEN dumping to JSON with pretty-option enabled
json = dump_json(data, pretty=True)
# THEN the output is formatted over multiple lines
assert isinstance(json, str)
assert len(json.split('\n')) == 4
def blockinfo(IP):
###Needs to be simplified with JSON
import urllib,urllib2,json,re
test = re.search("(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)",IP)
if test == None:#user
passdict = {"action":"query", "list":"blocks", "bkprop":"id|user|by|timestamp|expiry|reason|range|flags", "bklimit":"1","bkusers":IP}
else:#ip
passdict = {"action":"query", "list":"blocks", "bkprop":"id|user|by|timestamp|expiry|reason|range|flags", "bklimit":"1","bkip":IP}
urldata = urllib.urlencode(passdict)
baseurl = "http://en.wikipedia.org/w/api.php"
url = str(baseurl) + "?" + str(urldata)
urlobj = urllib2.urlopen(url)
json = urlobj.read()
urlobj.close()
try:json = json.split("<span style=\"color:blue;\"><blocks></span>")[1]
except:return "User is not blocked."
json = json.split("<span style=\"color:blue;\"></blocks></span>")[0]
json = json.split("<span style=\"color:blue;\"><")[1]
json = json.replace(""","\"")
bid=json.split("block id=\"")[1]
bid=bid.split("\"")[0]
gen="Block " + str(bid)
bid=json.split("user=\"")[1]
bid=bid.split("\"")[0]
gen=gen+" targeting " + str(bid)
bid=json.split("by=\"")[1]
bid=bid.split("\"")[0]
gen=gen+" was blocked by " + str(bid)
bid=json.split("timestamp=\"")[1]
bid=bid.split("\"")[0]
gen=gen+" @" + str(bid)
bid=json.split("expiry=\"")[1]
bid=bid.split("\"")[0]
gen=gen+" and expires at " + str(bid)
bid=json.split("reason=\"")[1]
bid=bid.split("\"")[0]
gen=gen+" because \"" + str(bid) + "\" ("
gen = gen.replace("&lt;","<")
gen = gen.replace("&gt;",">")
if "nocreate" in json:
gen = gen + "Account Creation Blocked, "
if "anononly" in json:
gen = gen + "Anonomous Users Only, "
else:
gen = gen + "Hardblocked, "
if not "allowusertalk" in json:
gen = gen + "User Talk Page REVOKED)"
else:
gen = gen + "User Talk Page allowed)"
return gen
def main():
jsons = glob.glob(BasePath + "test_images_jsons/*.json")
for json in tqdm(jsons):
data = read_json(json)
# print(data)
saveResultsByJson(
BasePath + "ali_test_images_jsons/{}".format(json.split("/")[-1]),
data)
def __compare_datetime(self, datefilter, json):
"""
Compares two date-time values to determine if an episode was published before or after the given date-time.
"""
publish_date = datetime.strptime(json.split('T')[0], '%Y-%m-%d')
filter_date = datetime.strptime(datefilter, '%Y-%m-%d')
result = publish_date > filter_date
return result
def __call__(self):
if not os.path.exists('./data/train_binary'):
os.mkdir('./data/train_binary')
if not os.path.exists('./data/cluster'):
os.mkdir('./data/cluster')
if not os.path.exists('./data/LaneImages'):
os.mkdir('./data/LaneImages')
jsons = [
json for json in os.listdir(self.tusimple)
if json.split('.')[-1] == 'json'
]
for j in jsons:
data = []
with open(os.path.join(self.tusimple, j)) as f:
for line in f.readlines():
data.append(json.loads(line))
for entry in data:
height = entry['h_samples']
width = entry['lanes']
clip = entry['raw_file']
img = cv2.imread(os.path.join(self.tusimple, clip))
cv2.imwrite(
os.path.join('./data/LaneImages',
'_'.join(clip.split('/')[1:])), img)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img_binary = np.zeros(img.shape, dtype=np.uint8)
img_cluster = np.zeros(img.shape, dtype=np.uint8)
for lane in range(len(width)):
coordinate = []
for w, h in zip(width[lane], height):
if w == -2:
continue
else:
coordinate.append(np.array([w, h], dtype=np.int32))
img_binary = cv2.polylines(img_binary,
np.stack([coordinate]),
isClosed=False,
color=255,
thickness=5)
img_cluster = cv2.polylines(img_cluster,
np.stack([coordinate]),
isClosed=False,
color=255 - lane * 50,
thickness=5)
name_list = clip.split('/')[1:]
new_name = '_'.join(name_list)
new_name = '.'.join([new_name.split('.')[0], 'png'])
img_binary = self.transform(img_binary)
img_cluster = self.transform(img_cluster, target='instance')
cv2.imwrite(os.path.join('./data/train_binary', new_name),
img_binary)
cv2.imwrite(os.path.join('./data/cluster', new_name),
img_cluster)
def send_payload_ok(json, link):
serverId = json.split('serverId')[1].split(': u\'')[1].split('\'')[0]
print('[i] serverId: ' + serverId)
serverUrl = json.split('serverUrl')[1].split(': u\'')[1].split('\'')[0]
print('[i] serverUrl: ' + serverUrl)
id_process = json.split('id_process')[1].split(': u\'')[1].split('\'')[0]
print('[i] id_process: ' + id_process)
payloadjson_processVideo = {
'function': 'processVideo',
'args[dummy]': '1',
'args[urlEntryUser]': str(link),
'args[fromConvert]': 'urlconverter',
'args[requestExt]': str(args.format),
'args[serverId]': str(serverId),
'args[nbRetry]': '0',
# 'args[title]': '0',
'args[serverUrl]': str(serverUrl),
'args[id_process]': str(id_process),
'args[videoResolution]': '-1',
'args[audioBitrate]': str(args.bitrate),
'args[audioFrequency]': '0',
'args[channel]': 'stereo',
'args[volume]': '0',
'args[startFrom]': '-1',
'args[endTo]': '-1',
'args[custom_resx]': '-1',
'args[custom_resy]': '-1',
'args[advSettings]': 'true',
'args[aspectRatio]': '-1'
}
print(payloadjson_processVideo)
print('\n[+] sending POST request OK...')
post = session.post(convert_url,
data=payloadjson_processVideo,
headers=headers2)
print('[+] success.')
print('[+] checking id...')
jsonpostok = str(post.json())
print('\n\n' + jsonpostok + '\n\n')
getcheck_id(jsonpostok, link)
def convertFromJsonToDate(json):
if (json is not None):
date_str = json.split('-')
date_valid = date(
int(date_str[0]),
int(date_str[1]),
int(date_str[2][:2]))
date_valid = date_valid + timedelta(days=1)
return date_valid
return None
def process_json(json):
# split into array because of additional information
json_processed = json.split(' ')[0]
# only extract digits from processed json
result = ''.join([i for i in json_processed if i.isdigit()])
#try to cast result to float, if fails, return 0 because there is no value from imdb
try:
return float(result)
except ValueError:
return 0
def main():
jsons = glob.glob(
"/Users/lingmou/Desktop/python-script/20ImageRecongnize/resource/jsons/*.json"
)
for json in jsons:
bboxes = read_json(json)
for item in bboxes:
item["x"] = int((item["x1"] + item["x2"]) / 2)
item["y"] = int((item["y1"] + item["y2"]) / 2)
saveResultsByJson(
"/Users/lingmou/Desktop/python-script/20ImageRecongnize/centerpoint/{}"
.format(json.split("/")[-1]), bboxes)
def __call__(self):
if not os.path.exists('./data/train_binary'):
os.mkdir('./data/train_binary')
if not os.path.exists('./data/cluster'):
os.mkdir('./data/cluster')
if not os.path.exists('./data/LaneImages'):
os.mkdir('./data/LaneImages')
jsons = [
json for json in os.listdir(self.tusimple)
if json.split('.')[-1] == 'json'
]
for j in jsons:
data = []
with open(os.path.join(self.tusimple, j)) as f:
for line in f.readlines():
data.append(json.loads(line))
for entry in data:
height = entry['h_samples']
width = entry['lanes']
for index, w in enumerate(width):
counter = list(set(w))
if counter[0] == -2 and len(counter) == 1:
del width[index]
clip = entry['raw_file']
img = cv2.imread(os.path.join(self.tusimple, clip))
cv2.imwrite(
os.path.join('./data/LaneImages',
'_'.join(clip.split('/')[1:])), img)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img_binary = np.zeros_like(img)
img_cluster = np.zeros_like(img)
for lane in range(len(width)):
queue = []
for h, w in zip(height, width[lane]):
if w < 0:
continue
else:
queue.insert(0, (w, h))
if len(queue) == 2:
cv2.line(img_binary, queue[0], queue[1], 255,
self.line_width)
cv2.line(img_cluster, queue[0], queue[1],
255 - 50 * lane, self.line_width)
if len(queue) > 1:
queue.pop()
new_name = '_'.join(clip.split('/')[1:])
new_name = '.'.join([new_name.split('.')[0], 'png'])
cv2.imwrite(os.path.join('./data/train_binary', new_name),
img_binary)
cv2.imwrite(os.path.join('./data/cluster', new_name),
img_cluster)
def convert_pascal_to_tfrecord():
json_path = FLAGS.data_dir + FLAGS.json_dir
image_path = FLAGS.data_dir + FLAGS.image_dir
save_path = FLAGS.save_dir + cfgs.VERSION + '_' + FLAGS.dataset + '_' + FLAGS.save_name + '.tfrecord'
print(save_path)
mkdir(FLAGS.save_dir)
# writer_options = tf.python_io.TFRecordOptions(tf.python_io.TFRecordCompressionType.ZLIB)
# writer = tf.python_io.TFRecordWriter(path=save_path, options=writer_options)
writer = tf.python_io.TFRecordWriter(path=save_path)
# print(json_path)
for count, json in enumerate(glob.glob(json_path + '/*.json')):
print(json)
# to avoid path error in different development platform
json = json.replace('\\', '/')
img_name = json.split('/')[-1].split('.')[0] + FLAGS.img_format
img_path = image_path + '/' + img_name
if not os.path.exists(img_path):
print('{} is not exist!'.format(img_path))
continue
# read_json_gtbox_and_label(json)
img_height, img_width, gtbox_label = read_json_gtbox_and_label(json)
print('gtbox shape' ,gtbox_label.shape)
print('gtbox ' ,gtbox_label)
if gtbox_label.shape[0] == 0:
continue
# img = np.array(Image.open(img_path))
img = cv2.imread(img_path)
feature = tf.train.Features(feature={
# do not need encode() in linux
'img_name': _bytes_feature(img_name.encode()),
# 'img_name': _bytes_feature(img_name),
'img_height': _int64_feature(img_height),
'img_width': _int64_feature(img_width),
'img': _bytes_feature(img.tostring()),
'gtboxes_and_label': _bytes_feature(gtbox_label.tostring()),
'num_objects': _int64_feature(gtbox_label.shape[0])
})
example = tf.train.Example(features=feature)
writer.write(example.SerializeToString())
view_bar('Conversion progress', count + 1, len(glob.glob(json_path + '/*.json')))
# print(label_list)
print('\nConversion is complete!')
def get_STgraphs(jsonList, seuil, ratio, only_xy=False, test=False):
"""Retourne la list des noeuds, arcs et y de la list de JSON"""
Nodes, Edges, Y = list(), list(), list()
path = "/home/nmiguens/JSON/WLASL/"
for json in jsonList:
if test:
node, adj = dict_to_STgraph(json_to_dict(path + json), seuil,
ratio, 1)
mean, var = json_to_MV(json)
node = (node - mean) / np.sqrt(var + 1e-10)
y = json.split("_")[0]
# adj = normalize(adj)
if only_xy:
Nodes.append(node[:, :2])
else:
Nodes.append(node)
Edges.append(adj)
Y.append(y)
else:
for modulo in range(ratio):
node, adj = dict_to_STgraph(json_to_dict(path + json), seuil,
ratio, modulo)
mean, var = json_to_MV(json)
node = (node - mean) / np.sqrt(var + 1e-10)
y = json.split("_")[0]
if only_xy:
Nodes.append(node[:, :2])
else:
Nodes.append(node)
Edges.append(adj)
Y.append(y)
Y = np.array(Y)
unique_labels = np.unique(Y)
Y = [np.where(target == unique_labels)[0][0] for target in Y]
return Nodes, Edges, Y
def get_STgraphs(jsonList, seuil):
"""Retourne la list des noeuds, arcs et y de la list de JSON"""
Nodes, Y = list(), list()
path = "/home/nmiguens/JSON/WLASL/"
for json in jsonList :
node = dict_to_STgraph(json_to_dict(path + json), seuil)
mean, var = json_to_MV(json)
node[:,0:2] = (node[:,0:2] - mean[0:2])/np.sqrt(var[0:2] + 1e-10)
Nodes.append(node)
y = json.split("_")[0]
Y.append(y)
Y = np.array(Y)
unique_labels = np.unique(Y)
Y = [np.where(target == unique_labels)[0][0]
for target in Y]
return Nodes, Y, unique_labels
def strobj(json, obj) :
v = json.split("=");
if len(v) != 2 :
return {}
names = v[0].split(".")
value = v[1]
subjson = ''
if len(names) > 1 :
if re.findall("[a-z]", names[1]) :
if names[0] == '0' :
if len(obj) == 0 :
obj.append({})
else :
if not obj.has_key(names[0]) :
obj[names[0]] = {}
else :
if names[0] == '0' :
if len(obj) == 0 :
obj.append([])
else :
if not obj.has_key(names[0]) :
obj[names[0]] = []
curname = names[0]
del[names[0]]
subjson = '.'.join(names) + "=" + value
if curname == '0' :
curname = 0
obj[curname] = strobj(subjson, obj[curname])
else :
#print curr_type
if curr_type == 'int' :
if names[0] == '0' :
obj.append(int(value))
else :
obj[names[0].replace("^\s+|\s+", '')] = int(value)
else :
if names[0] == '0' :
obj.append(value)
else :
obj[names[0].replace("^\s+|\s+", '')] = value
return obj
def main():
# images = glob.glob(imagesPath + "*.jpg")
indexChangeRecord = []
index = 7900001
jsons = glob.glob(BasePath + "oldJsons/*.json")
images = glob.glob(BasePath + "aImages/*.jpg")
for json in tqdm(jsons):
for image in images:
if image.split("/")[-1].split(".")[0] == json.split("/")[-1].split(".")[0]:
indexObject = {
"old": image.split("/")[-1].split(".")[0],
"new": index
}
indexChangeRecord.append(indexObject)
shutil.copy(image, BasePath + "target/images/{}.jpg".format(str(index)))
shutil.copy(json, BasePath + "target/jsons/{}.json".format(str(index)))
index += 1
saveResultsByJson("indexChangeRecord", indexChangeRecord)
def parse_jsons(json_directory):
pacs = {}
pans = {}
anum = {}
columns = []
jsons = os.listdir(json_directory)
for json in jsons:
variant = json.split('.')[0]
jpath = json_directory + '/' + json
jdata = load_json(jpath)
if 'pop_acs' in jdata.keys() and 'pop_ans' in jdata.keys() and 'allele_num' in jdata.keys():
pacs[variant] = jdata['pop_acs']
pans[variant] = jdata['pop_ans']
anum[variant] = jdata['allele_num']
for key in jdata['pop_acs']:
columns.append(key)
else:
pacs[variant] = 'No Annnotations Available'
pans[variant] = 'No Annnotations Available'
anum[variant] = 'No Annnotations Available'
return pacs, pans, anum, list(set(columns))
def seanomeReadCount():
abundanceFilePath = "coverage_json_files"
files = glob.glob(abundanceFilePath + "/*.json")
allSeanome = []
for file in files:
sample = file.split(".")[0]
with open(file, "r") as json:
json = json.read()
lines = json.split("[")
histoData = []
for term in lines[3:]:
newTerm = term.split("]")[0].split(",")
depth = int(newTerm[0])
abundance = int(newTerm[1])
for x in range(abundance):
histoData.append(depth)
allSeanome.append(histoData)
return allSeanome
def get_output_as_list_dict(self, json):
'''
It converts the JSON string retrieval into a list of dictionaries where each dictionary represent a retrieval
:param json: String in a JSON format
:return: List of dictionaries
'''
output = []
docs = json.split(', {')
i = 0
for doc in docs:
i += 1
if len(docs) == 1:
if (doc == '[]'):
print('Your search did not match any documents.')
else:
output.append(ast.literal_eval(doc[1:-1]))
elif i == 1:
output.append(ast.literal_eval(doc[1:]))
elif i < len(docs):
output.append(ast.literal_eval('{' + doc))
else:
output.append(ast.literal_eval('{' + doc[:-1]))
return output
def update_max_token_len(html, json, max_len):
html_len, json_len = len(html.split()), len(json.split())
return max(html_len, max(json_len, max_len))
file_year=datetime.datetime.fromtimestamp(os.path.getmtime(file_name)).year
file=open(file_name,'r')
for line in file:
line=line.strip()
line=line.replace('invalid user ','')
if 'ssh' in line:
times=1
repeat_str='message repeated '
repeat_ind=line.find(repeat_str)
if repeat_ind>=0:
repeat_bkt=line.find('[',repeat_ind)
if repeat_bkt>=0 and repeat_ind<repeat_bkt:
times=int(re.search(r'\d+',line[repeat_ind:-1]).group())
line=line[0:repeat_ind]+line[repeat_bkt+2:-1]
parse_ssh_entry(line,file_year)
json=json.dumps({
'users_f':users_f,
'users_p':users_p,
'ip_f':ip_f,
'ip_p':ip_p,
'country_f':country_f,
'country_p':country_p,
'gps_f':gps_f,
'gps_p':gps_p})
json_str=''.join(json.split())
print(json_str)
exit(0)
except Exception as error:
print(error)
exit(1)
for line in file:
line = line.strip()
line = line.replace('invalid user ', '')
if 'ssh' in line:
times = 1
repeat_str = 'message repeated '
repeat_ind = line.find(repeat_str)
if repeat_ind >= 0:
repeat_bkt = line.find('[', repeat_ind)
if repeat_bkt >= 0 and repeat_ind < repeat_bkt:
times = int(
re.search(r'\d+', line[repeat_ind:-1]).group())
line = line[0:repeat_ind] + line[(repeat_bkt + 2):-1]
parse_ssh_entry(line, file_year)
json = json.dumps({
'users_f': users_f,
'users_p': users_p,
'ip_f': ip_f,
'ip_p': ip_p,
'country_f': country_f,
'country_p': country_p,
'gps_f': gps_f,
'gps_p': gps_p
})
json_str = ''.join(json.split())
print(json_str)
sys.exit(0)
except KeyboardInterrupt:
sys.exit(-1)
def sanitize_json(json):
json = json.replace("\'", "\"")
json = json.split('[')[1].split(']')[0]
json = json[0:len(json) - 6] + "}"
return json