def survey_metadata(filename):
ogg_db = sqlite3.connect(filename)
ogg_cursor = ogg_db.execute('SELECT * FROM image where img_minor_mime="ogg"')
_broken_metadata = {}
start_time = time.time()
metadata_dicts = []
streams_dicts = []
i = 0
for record in ogg_cursor:
name = record[0]
metadata_string = record[4]
try:
metadata = phpserialize.loads(metadata_string.replace(u'\\"', u'\"'))
except:
try:
metadata = phpserialize.loads(metadata_string.encode('utf-8').decode('unicode-escape'))
except:
_broken_metadata[name] = metadata_string
continue
streams = metadata.pop('streams', {})
streams_dicts.extend(streams.values())
metadata_dicts.append(metadata)
i += 1
if i % 100 == 0:
cur_dur = time.time() - start_time
print i, 'records complete in', round(cur_dur, 2), 'seconds.',
print len(_broken_metadata), 'records skipped.'
return {'general': survey_dicts(metadata_dicts),
'streams': survey_dicts(streams_dicts)}
def download_additional_image_sizes(self, item, wordpress_namespace,
source_path):
if phpserialize is None:
return
additional_metadata = item.findall(
'{{{0}}}postmeta'.format(wordpress_namespace))
if additional_metadata is None:
return
for element in additional_metadata:
meta_key = element.find(
'{{{0}}}meta_key'.format(wordpress_namespace))
if meta_key is not None and meta_key.text == '_wp_attachment_metadata':
meta_value = element.find(
'{{{0}}}meta_value'.format(wordpress_namespace))
if meta_value is None:
continue
# Someone from Wordpress thought it was a good idea
# serialize PHP objects into that metadata field. Given
# that the export should give you the power to insert
# your blogging into another site or system its not.
# Why don't they just use JSON?
if sys.version_info[0] == 2:
try:
metadata = phpserialize.loads(
utils.sys_encode(meta_value.text))
except ValueError:
# local encoding might be wrong sometimes
metadata = phpserialize.loads(
meta_value.text.encode('utf-8'))
else:
metadata = phpserialize.loads(
meta_value.text.encode('utf-8'))
size_key = b'sizes'
file_key = b'file'
if size_key not in metadata:
continue
for filename in [
metadata[size_key][size][file_key]
for size in metadata[size_key]
]:
url = '/'.join([source_path, filename.decode('utf-8')])
path = urlparse(url).path
dst_path = os.path.join(*([self.output_folder, 'files'] +
list(path.split('/'))))
dst_dir = os.path.dirname(dst_path)
utils.makedirs(dst_dir)
LOGGER.info("Downloading {0} => {1}".format(url, dst_path))
self.download_url_content_to_file(url, dst_path)
dst_url = '/'.join(dst_path.split(os.sep)[2:])
links[url] = '/' + dst_url
links[url] = '/' + dst_url
def extract_wp_attachments(self, node):
attachments = []
for attachment_url in node.iterfind(WP + 'attachment_url'):
attachment = dict(url=attachment_url.text, files=[], metadata=[],
backups=[])
for filename in get_meta_values_by_key(node, '_wp_attached_file'):
attachment['files'].append(filename)
for metadata in get_meta_values_by_key(node, '_wp_attachment_metadata'):
attachment['metadata'].append(phpserialize.loads(metadata))
for bkp in get_meta_values_by_key(node, '_wp_attachment_backup_sizes'):
attachment['backups'].append(phpserialize.loads(bkp))
attachments.append(attachment)
return attachments
def download_additional_image_sizes(self, item, wordpress_namespace, source_path):
if phpserialize is None:
return
additional_metadata = item.findall('{{{0}}}postmeta'.format(wordpress_namespace))
if additional_metadata is None:
return
for element in additional_metadata:
meta_key = element.find('{{{0}}}meta_key'.format(wordpress_namespace))
if meta_key is not None and meta_key.text == '_wp_attachment_metadata':
meta_value = element.find('{{{0}}}meta_value'.format(wordpress_namespace))
if meta_value is None:
continue
# Someone from Wordpress thought it was a good idea
# serialize PHP objects into that metadata field. Given
# that the export should give you the power to insert
# your blogging into another site or system its not.
# Why don't they just use JSON?
if sys.version_info[0] == 2:
try:
metadata = phpserialize.loads(utils.sys_encode(meta_value.text))
except ValueError:
# local encoding might be wrong sometimes
metadata = phpserialize.loads(meta_value.text.encode('utf-8'))
else:
metadata = phpserialize.loads(meta_value.text.encode('utf-8'))
size_key = b'sizes'
file_key = b'file'
if size_key not in metadata:
continue
for filename in [metadata[size_key][size][file_key] for size in metadata[size_key]]:
url = '/'.join([source_path, filename.decode('utf-8')])
path = urlparse(url).path
dst_path = os.path.join(*([self.output_folder, 'files']
+ list(path.split('/'))))
dst_dir = os.path.dirname(dst_path)
utils.makedirs(dst_dir)
LOGGER.info("Downloading {0} => {1}".format(url, dst_path))
self.download_url_content_to_file(url, dst_path)
dst_url = '/'.join(dst_path.split(os.sep)[2:])
links[url] = '/' + dst_url
links[url] = '/' + dst_url
def roles(self):
"""Returns a list of all roles for the user."""
key = WORDPRESS_TABLE_PREFIX + 'capabilities'
roles = self.meta.get(meta_key=key).meta_value
roles = phpserialize.loads(force_bytes(roles), decode_strings=True)
return [role for role, enabled in roles.items() if enabled]
def authenticate(realm, authid, details):
print(
"WAMP-Anonymous dynamic authenticator invoked: realm='{}', authid='{}'"
.format(realm, authid))
ticket = details['ticket']
# pprint(details)
data = load_token(ticket)
if data is None:
raise ApplicationError(
u"no_such_user",
u"could not authenticate session - no such principal {}".
format(authid))
ticket = loads(data)
# ticket holds 5 values!!!
user_id, login, password, roles, is_enabled = ticket.values()
if authid == login:
principal = {
u'role': u'public',
u'extra': {
u'session': details['session']
}
}
return principal
else:
error = u"could not authenticate session - invalid ticket '{}' for principal {}".format(
ticket, authid)
raise ApplicationError(u"invalid_ticket", error)
def extract_user_id(self):
session = phpserialize.loads(base64.b64decode(self.payload), object_hook=php_object_hook, decode_strings=True)
for key, value in session.items():
if key.startswith("login_"):
return value
return None
def roles(self):
"""Returns a list of all roles for the user."""
key = WORDPRESS_TABLE_PREFIX + 'capabilities'
roles = self.meta.get(meta_key=key).meta_value
roles = phpserialize.loads(force_bytes(roles), decode_strings=True)
return [role for role, enabled in roles.items() if enabled]
def _request(self, path, signature, params=None, result="xml"):
"""
Common request functionality for Services and Contacts
- makes request
- parses xml
- handles errors
"""
headers = {"Content-type": "application/xml", "X_USER": self.oi_username, "X_SIGNATURE": signature}
conn = httplib.HTTPConnection(self.api_domain)
params_string = params or ""
conn.request("POST", path, params_string, headers)
response = conn.getresponse()
data = response.read()
conn.close()
if result == "xml":
data_decompressed = self._decompress_xml(data).decode("utf8")
parsed_data = self._parse_data(data_decompressed)
if parsed_data.error != "OK":
self._handle_error(parsed_data.error)
if result == "php":
parsed_data = phpserialize.loads(data)
return parsed_data
def loads(cls, data): try: if isinstance(data, str): data = data.encode() return _phpsa.loads(data, decode_strings=True) except: return pickle.loads(data)
def parse_permissions(permission_array):
"""Parses a python dict of lists from the forum permission array.
The forum permissions are very creative saved in the database. Its a
php-serialized associative array that holds the different permission
mask ids as string-value for the permission type key. The single ids
are separated by commas.
This function parses this to a dict with the permission types as keys
and a list of the mask ids as value.
:type permission_array: str or bytes
:param permission_array: The phpserializes crippled arraystring
:rtype: dict[str, set[bytes]]
:return: The parsed dict with the permissions.
"""
parsed = {}
if not len(permission_array):
return parsed
if isinstance(permission_array, str):
permission_array = bytes(permission_array, "utf8")
perms = phpserialize.loads(permission_array, decode_strings=True)
for perm_type in perms:
parsed[perm_type] = set()
if perms[perm_type] is not None:
for mask in perms[perm_type].split(","):
try:
parsed[perm_type].add(int(mask))
except ValueError:
pass
return parsed
def process_db_result(raw_data):
data = {}
# remove non-useful keys to other dbs
for key in {
"fa_id",
"fa_deleted_user",
"fa_deleted_reason_id",
"fa_description_id",
"fa_actor",
}:
raw_data.pop(key, None)
# Metadata can be stored as JSON or as a serialized PHP array.
metadata = raw_data.pop("fa_metadata", "")
try:
data["fa_metadata"] = json.loads(metadata)
except json.JSONDecodeError:
data["fa_metadata"] = phpserialize.loads(metadata, decode_strings=True)
components = data["fa_metadata"].get("ComponentsConfiguration", {})
if components:
components.pop("_type", "")
data["fa_metadata"][
"ComponentsConfiguration"] = phpserialize.dict_to_list(
components)
# stringify everything else
for key, value in raw_data.items():
if not value:
continue
elif isinstance(value, bytes):
# decode unicode strings
data[key] = str(value, encoding="utf-8")
else:
data[key] = str(value)
return data
def get_redirect(params: str):
try:
params = phpserialize.loads(params.encode('utf-8'))
redirect = params[b'4::target'].decode('utf-8')
return redirect
except ValueError:
return params
def save_attachment(self):
if not self.parent_key:
return
parent = models.Post.get_by_key_name(self.parent_key)
if not parent:
return
att = models.Attachment.get_or_insert(
str(self.id),
parent=parent,
)
att.title = self.title
att.filename = self.url
att.put()
metadata = self.postmeta.get('_wp_attachment_metadata').encode('utf-8')
data = php.loads(metadata)
if 'sizes' in data:
for key, val in data['sizes'].items():
var = models.ImageVariant.get_or_insert(
parent=att, key_name=key)
var.width = int(val['width'])
var.height = int(val['height'])
var.filename = val['file']
var.save()
def getCacheSizes():
postfixes = []
themes = args.themes
cur = db.cursor(MySQLdb.cursors.DictCursor)
numrows = cur.execute(
"SELECT aux, data FROM " + table_prefix +
"plugin_storage WHERE type='cacheManager' ORDER BY aux")
themes.append(getCurrentTheme())
themes.append('admin')
for x in range(0, numrows):
row = cur.fetchone()
themeName = row['aux']
data = phpserialize.loads(row['data'].encode(), decode_strings=True)
postfix_string = (
('_' + str(data['image_size']) if data['image_size'] else '') +
('_w' + str(data['image_width']) if data['image_width'] else '') +
('_h' + str(data['image_height']) if data['image_height'] else '')
+ ('_cw' + str(data['crop_width']) if data['crop_width'] else '') +
('_ch' + str(data['crop_height']) if data['crop_height'] else '') +
('_cx' + str(data['crop_x']) if data['crop_x'] else '') +
('_cy' + str(data['crop_y']) if data['crop_y'] else '') +
('_thumb' if data['thumb'] else '') +
('_' + str(data['wmk']) if data['wmk'] else '') +
('_' + str(data['gray']) if data['gray'] else ''))
if (themeName in themes):
postfixes.append(postfix_string)
print('will cache ' + str(len(postfixes)) + ' sizes for \'' +
'\', \''.join(themes) + '\'...')
return postfixes
def getCurrentTheme():
cur = db.cursor(MySQLdb.cursors.DictCursor)
options = cur.execute("SELECT value FROM " + table_prefix +
"options WHERE name='gallery_data'")
data = phpserialize.loads(cur.fetchone()['value'].encode(),
decode_strings=True)
return data['current_theme']
def test_basic_object_hook(self):
data = b'O:7:"WP_User":1:{s:8:"username";s:5:"admin";}'
user = phpserialize.loads(data,
object_hook=phpserialize.phpobject,
decode_strings=True)
self.assertEqual(user.username, 'admin')
self.assertEqual(user.__name__, 'WP_User')
def _request(self, path, signature, params=None, result='xml'):
'''
Common request functionality for Services and Contacts
- makes request
- parses xml
- handles errors
'''
headers = {'Content-type': 'application/xml','X_USER': USERNAME, 'X_SIGNATURE': signature}
conn = httplib.HTTPConnection(self.api_domain)
params_string = params or ''
conn.request('POST', path, params_string, headers)
response = conn.getresponse()
data = response.read()
conn.close()
if result == 'xml':
data_decompressed = self._decompress_xml(data).decode('utf8')
parsed_data = self._parse_data(data_decompressed)
if parsed_data.error != 'OK':
self._handle_error(parsed_data.error)
if result == 'php':
parsed_data = phpserialize.loads(data)
return parsed_data
def _maybe_php_deserialize(value):
try:
deserialized_value = phpserialize.loads(value)
except ValueError:
return value
else:
return _neaten_php_value(deserialized_value)
def restriction(self, obj):
#restrictions =
res = ''
restrict_catalog = ['', '', 'disabled="disabled"']
restrict_product = ['', '', 'disabled="disabled"']
if obj.restrictions:
res = 'checked="checked"'
restrictions = phpserialize.loads(obj.restrictions)
if ('restrict_catalog' in restrictions.keys()):
restrict_catalog[0] = 'checked="checked"'
restrict_catalog[1] = restrictions['restrict_catalog'][0]
restrict_catalog[2] = ''
if ('restrict_product' in restrictions.keys()):
restrict_product[0] = 'checked="checked"'
restrict_product[1] = restrictions['restrict_product'][0]
restrict_product[2] = ''
out = ""
out += '<div class="wl-radio">'
out += '<div class="radio-inline"><input type="checkbox" id="is_restrict" name="is_restrict" value="1" ' + res + '/><p class="mr10 control-static">分类/产品限制</p></div>'
out += '<div class="radio-inline"><input name="restrictions" value="restrict_catalog" type="radio" id="restriction_catalog" ' + restrict_catalog[
0] + '/><p class="mr10 control-static">分类限制</p><input class="mr10" name="restrict_catalog" type="text" class="inline numeric" ' + restrict_catalog[
2] + ' value="' + restrict_catalog[
1] + '" /><p class="mr10 control-static"> (分类id(catalog_id),多个用","隔开)</p></div>'
out += '<div class="radio-inline"><input name="restrictions" value="restrict_product" type="radio" id="restriction_product" ' + restrict_product[
0] + '/><p class="mr10 control-static">产品限制</p><input class="mr10" name="restrict_product" type="text" class="inline" ' + restrict_product[
2] + ' value="' + restrict_product[
1] + '" /> <p class="mr10 control-static">(产品SKU,多个用","隔开) </p></div>'
out += '</div>'
return out
def try_to_unserialise(value):
try:
return phpserialize.loads(value)
except Exception, ex:
print value
print
raise CommandError(str(ex))
def get_ebay_shipping_fee_from_ibay(self, item_id):
sql = ("SELECT sku,shippingdetails,ei.itemid FROM ebay_item ei "
"LEFT JOIN ebay_item_detail eid ON ei.itemid=eid.itemid "
"WHERE ei.itemid = %s")
out = dict()
try:
self.ibay_cur.execute(sql, (str(item_id),))
ret = self.ibay_cur.fetchone()
if ret:
sku = ret[0].split('@#')[0]
out['sku'] = sku
shipping_str = bytes(ret[1], encoding='utf8')
shipping = phpserialize.loads(shipping_str)
# print(shipping)
try:
item = shipping[b'ShippingServiceOptions']
for k in item:
if item[k][b'ShippingServicePriority'].decode('utf-8') == '1':
try:
out['shipping_fee'] = item[k][b'ShippingServiceAdditionalCost'].decode('utf-8')
except BaseException:
out['shipping_fee'] = item[k][b'ShippingServiceCost'].decode('utf-8')
out['shipping_name'] = item[k][b'ShippingService'].decode('utf-8')
break
except BaseException:
out['shipping_fee'] = 0
out['shipping_name'] = ''
return out
except Exception as e:
self.logger.error(f"Failed to get shipping fee of item {item_id} cause of {e}")
return out
def get_redirect(params: str):
try:
params = phpserialize.loads(params.encode('utf-8'))
redirect = params[b'4::target'].decode('utf-8')
return redirect
except ValueError:
return params
def index():
serialized = request.form.get('serialized', '')
unserialized = None
try:
unserialized = phpserialize.loads(serialized)
except ValueError:
pass
if not unserialized:
try:
unserialized = json.loads(serialized)
except ValueError:
pass
if not unserialized:
try:
unserialized = ast.literal_eval(serialized)
except (SyntaxError, ValueError):
pass
formatted_unserialized = ''
if unserialized:
formatted_unserialized = json.dumps(unserialized, indent=2)
return render_template('index.html',
formatted_unserialized=formatted_unserialized,
serialized=serialized
)
def parse_plugins_to_dict(phpdata):
""" Try to parse the serialized php objects of the plugins updater"""
from phpserialize import loads,phpobject
import time
try:
# deserialize the outer php class to a dict
p=loads(phpdata,object_hook=phpobject)
d1=p._asdict()
if 'checked' in d1:
checked = d1['checked']
else:
checked = {}
# yes,here they call the field response
if 'response' in d1:
response = d1['response']
else:
response = {}
rdict={}
for k,r in response.iteritems():
rdict[k]=r._asdict()
age = time.time() - d1['last_checked']
return age,checked,rdict
except Exception,e:
logger.error("Error parsing serialized php data: plugins")
logger.exception(e)
logger.error("'"+phpdata+"'")
def loads(self, data):
def object_hook(name, d):
# Need to lowercase these, because the data is inconsistent in how
# it names the classes
name = name.lower()
if name == 'album':
fields = d['fields']
return Album(name=fields['name'], title=fields.get('title'),
description=fields.get('description'),
parent=fields.get('parentAlbumName'))
elif name == 'albumitem':
# This contains info about the image and the thumbnail. But
# since we don't care about the thumbnail, just collapse this
# together with the contained image, which we already parsed
kwargs = {'caption': d['caption']}
if d['isAlbumName']:
return AlbumItem.SubAlbum(d['isAlbumName'],
caption=d['caption'])
else:
image = d['image']
captured_at=datetime.date.fromtimestamp(
d['itemCaptureDate'][0])
return AlbumItem.Photo(image.name,
caption=d['caption'],
image_type=image.image_type,
captured_at=captured_at)
elif name == 'image':
return Photo(name=d['name'], image_type=d['type'])
loaded = phpserialize.loads(data, object_hook=object_hook)
if isinstance(loaded, dict):
loaded = phpserialize.dict_to_list(loaded)
return loaded
def parse_themes_to_dict(phpdata):
""" Try to parse the serialized php objects of the plugins updater"""
from phpserialize import loads,phpobject
import time
# yes, they really have different formats in each of the settings...
# while in core and plugins the response entry are objects, here they are
# already dicts... *sigh*
# At this point it was already too late to stop this little project...
# after wasting 8 hours into this...
try:
d1=loads(phpdata,object_hook=phpobject)._asdict()
checked = {}
response = {}
if 'checked' in d1:
checked = d1['checked']
if 'response' in d1:
response = d1['response']
if 'last_checked' in d1:
last_checked = d1['last_checked']
age = time.time() - last_checked
else:
age = -1
return age,checked,response
except Exception,e:
logger.error("Error parsing serialized php data")
logger.exception(e)
logger.error("'"+phpdata+"'")
def try_to_unserialise(value):
try:
return phpserialize.loads(value)
except Exception, ex:
print value
print
raise CommandError(str(ex))
def _get_tags(self, post_meta, post_title=None):
"""
Get tags related to a post
"""
# Unclassified posts will get a special tag.
# This will make it easier to find them afterwards.
tags = ["Unclassified"]
# First, unescape characters that were escaped to store the data in
# the backup file in CSV-like format
post_meta = post_meta.replace("\\", "")
if not post_meta:
logger.debug("post has no metadata: '%s'", post_title)
return tags
tags_dict = phpserialize.loads(post_meta.encode("utf-8"))
if not tags_dict:
logger.debug("post has really no tags: '%s'", post_title)
return tags
if b"tag" not in tags_dict:
logger.debug("post has no tags: '%s'", post_title)
return tags
tags = [tag.decode("utf-8") for tag in tags_dict[b"tag"].values()]
return tags
def get_balance_info(phone):
""" Get balance from connect API """
result = []
data = {'all_users': '1', 'phone': "8" + phone[1:]}
try:
users = phpserialize.loads(curl('get_user', data))
except ValueError:
users = None
if users:
for uid, user in users.iteritems():
#print user;
res = {
"id": user['IDUser'].encode('utf8', 'replace'),
"balance": user['Balance'].encode('utf8', 'replace'),
"is_locked": user['Status'] == -2,
}
# TODO
# next paymant date
#if user['Status'] == 1:
# date = strtotime($user['Service10000_Date']);
# res["paydate"] = date("Ymd", $date)."0000.00-0-000";
#else:
#paydate = "" #
# next payment summ
#user_info = phpserialize.loads(curl("get_user_nextpay", {"id": user['IDUser'], "get_next_pay": 1}))
#res["paydate"] = round(user_info[0]['next_payment'])
result.append(res)
# Закостылим для теста
if phone == '803':
#for i in [[200045, 1567],[223777, -2341.23],[300000, 0],[150230, 15023007000.30]]:
for i in [[200045, 1567]]:
result.append({'id': str(i[0]), 'balance': str(i[1])})
return result
def authenticate(self, session):
"""
Given a CakeSession object we'll authenticate it to an actual user.
"""
if (time() > session.expires or
not session.data.startswith('User|')):
session.delete()
return None
serialized_data = session.data[5:]
php_user = phpserialize.loads(serialized_data)
user_id = int(php_user.get('id'))
try:
profile = UserProfile.objects.get(pk=user_id)
except UserProfile.DoesNotExist:
session.delete()
return None
if profile.user is None:
profile.create_django_user()
return profile.user
def get_balance_info(phone):
""" Get balance from connect API """
result = []
data = {"all_users": "1", "phone": "8" + phone[1:]}
try:
users = phpserialize.loads(curl("get_user", data))
except ValueError:
users = None
if users:
for uid, user in users.iteritems():
# print user;
res = {
"id": user["IDUser"].encode("utf8", "replace"),
"balance": user["Balance"].encode("utf8", "replace"),
"is_locked": user["Status"] == -2,
}
# TODO
# next paymant date
# if user['Status'] == 1:
# date = strtotime($user['Service10000_Date']);
# res["paydate"] = date("Ymd", $date)."0000.00-0-000";
# else:
# paydate = "" #
# next payment summ
# user_info = phpserialize.loads(curl("get_user_nextpay", {"id": user['IDUser'], "get_next_pay": 1}))
# res["paydate"] = round(user_info[0]['next_payment'])
result.append(res)
# Закостылим для теста
if phone == "803":
# for i in [[200045, 1567],[223777, -2341.23],[300000, 0],[150230, 15023007000.30]]:
for i in [[200045, 1567]]:
result.append({"id": str(i[0]), "balance": str(i[1])})
return result
def get_partial_blocks():
sql = """
SELECT
log_params,
log_user_text as blocker,
log_title as blockee,
DATE_FORMAT( log_timestamp, '%Y-%m-%d') as date,
comment_text as reason
FROM logging_logindex l
LEFT JOIN comment on comment_id = log_comment_id
WHERE log_type = 'block'
AND log_params like '%"sitewide";b:0;%'
{0}
""".format(get_sql_time_frame('log_timestamp', '30 DAY'))
results = fetch_all(sql, cursor=DictCursor)
data = []
for block in results:
log_params = phpserialize.loads(block['log_params'],
decode_strings=True)
data.append([
block['blocker'].decode('utf-8'), block['blockee'].decode('utf-8'),
block['date'], block['reason'].decode('utf-8'),
json.dumps(log_params)
])
return data
def decrypt(self, payload):
data = json.loads(base64.b64decode(payload))
value = base64.b64decode(data['value'])
iv = base64.b64decode(data['iv'])
return loads(self._mcrypt_decrypt(value, iv))
def reload_system_config(self):
sql = 'SELECT `conf_Value` FROM `%s` WHERE `conf_Name`=\'system\' LIMIT 1' % self.ZBP_CONFIG
self.curs.execute(sql)
sys_conf = self.curs.fetchone()
sys_conf_dict = phpserialize.loads(sys_conf.encode())
for k in sys_conf_dict:
pass
def get_docs_labels():
pakets = Paket.query.all()
pakets = pakets_schema.dump(pakets)
docs = []
labels = []
names = []
for paket in pakets.data:
doc_paket = []
soals = Soal.query.filter_by(paket_id=paket['id'])
soals = soals_schema.dump(soals)
for soal in soals.data:
soal['pilihan'] = phpserialize.loads(
soal['pilihan'].encode('utf-8'), decode_strings=True)
text = [soal['soal']]
text.extend(list(soal['pilihan'].values()))
text = ' '.join(text)
doc_paket.append(text)
doc_paket = ' '.join(doc_paket)
docs.append(doc_paket)
labels.append(paket['id'])
names.append(paket['nama_paket'])
processor = TextPreprocessor()
data = []
for doc in docs:
temp = processor.clean(doc)
data.append(temp)
return data, labels, names
def RmItemFromSidebarWidget(Item='Meta'):
import phpserialize as ps
# Dump = b'a:3:{s:19:"wp_inactive_widgets";a:0:{}s:9:"sidebar-1";a:6:{i:0;s:8:"search-2";i:1;s:14:"recent-posts-2";i:2;s:17:"recent-comments-2";i:3;s:10:"archives-2";i:4;s:12:"categories-2";i:5;s:6:"meta-2";}s:13:"array_version";i:3;}'
Dump = get_option('sidebars_widgets')
ps.loads(Dump, decode_strings=True)
Loads = ps.loads(Dump, decode_strings=True)
# Loads['sidebar-1']
for k, v in Loads.items():
if k.startswith('sidebar'):
for Id, Widget in v.items():
print(Id, Widget)
if Widget.startswith('meta'):
del v[Id]
print('Deleted: ', Id, Widget)
break
# print(Loads)
update_option('sidebars_widgets', ps.dumps(Loads).decode())
def __decrypt_value(self):
# Decode the target cookie's json "value" and "iv" base64 encoded strings
encrypted_val = self.__decode_value()
# Decrypt the encrypted "value" raw bytes and parse the json "value" data into a dictionary
json_val = self.aes.decrypt(encrypted_val)
value_data = json.loads(json_val)
return phpserialize.loads(value_data['data'].encode())
def test_loads_dict(self):
self.assertEqual(
phpserialize.loads(b'a:3:{s:1:"a";i:1;s:1:"c";i:3;s:1:"b";i:2;}',
decode_strings=True), {
'a': 1,
'b': 2,
'c': 3
})
def get_namespaces(global_db, wiki_id):
""" Accesses the default content namespaces for the wiki
:param global_db: the global database object
"""
cursor = global_db.cursor()
results = cursor.execute("SELECT cv_value FROM city_variables WHERE cv_city_id = %s AND cv_variable_id = 359" % str(wiki_id))
result = cursor.fetchone()
return phpserialize.loads(result[0]).values() if result else [0, 14]
def download_additional_image_sizes(self, item, wordpress_namespace, source_path):
if phpserialize is None:
return
additional_metadata = item.findall("{{{0}}}postmeta".format(wordpress_namespace))
if additional_metadata is None:
return
for element in additional_metadata:
meta_key = element.find("{{{0}}}meta_key".format(wordpress_namespace))
if meta_key is not None and meta_key.text == "_wp_attachment_metadata":
meta_value = element.find("{{{0}}}meta_value".format(wordpress_namespace))
if meta_value is None:
continue
# Someone from Wordpress thought it was a good idea
# serialize PHP objects into that metadata field. Given
# that the export should give you the power to insert
# your blogging into another site or system its not.
# Why don't they just use JSON?
if sys.version_info[0] == 2:
metadata = phpserialize.loads(meta_value.text)
size_key = "sizes"
file_key = "file"
else:
metadata = phpserialize.loads(meta_value.text.encode("UTF-8"))
size_key = b"sizes"
file_key = b"file"
if not size_key in metadata:
continue
for filename in [metadata[size_key][size][file_key] for size in metadata[size_key]]:
url = "/".join([source_path, filename.decode("utf-8")])
path = urlparse(url).path
dst_path = os.path.join(*([self.output_folder, "files"] + list(path.split("/"))))
dst_dir = os.path.dirname(dst_path)
utils.makedirs(dst_dir)
LOGGER.notice("Downloading {0} => {1}".format(url, dst_path))
self.download_url_content_to_file(url, dst_path)
dst_url = "/".join(dst_path.split(os.sep)[2:])
links[url] = "/" + dst_url
links[url] = "/" + dst_url
def php2py(raw_php_var):
"""Convert a php code string into python object.
"""
python_var = phpserialize.loads(raw_php_var,
charset=encoding.default_encoding,
errors=encoding.default_errors,
decode_strings=True)
python_var = phpserialize_recursive_dict_to_list(python_var)
return python_var
def test_array_object(self):
self.assertEqual(phpserialize.loads(b"""a:1:{s:16:"525f70091c4bd_ja";C:11:"ArrayObject":896:{x:i:2;a:16:{s:10:"unit_count";i:3;s:7:"credits";d:0.14999999999999999;s:3:"eta";i:25236;s:8:"currency";s:3:"USD";s:4:"type";s:4:"text";s:15:"lc_src_detected";b:0;s:11:"custom_data";s:13:"525f70091c4bd";s:6:"lc_tgt";s:2:"ja";s:6:"lc_src";s:2:"en";s:5:"title";s:10:"sa dsa dsa";s:8:"body_src";s:10:"sa dsa dsa";s:10:"word_count";i:3;s:5:"force";i:1;s:7:"comment";s:0:"";s:12:"preview_html";s:463:"<div class='item type_text' id='cart_item_525f70091c4bd'>
<a href='javascript:;' class='remove_item'>x</a>
<span class='word_count'></span>
<p class='title'><strong>sa dsa dsa</strong></p>
<p class='body'>sa dsa dsa</p>
<p class='item_tools'><a href='javascript:;' class='edit_text'>Edit Text / Add Instructions</a></p>
</div>
";s:4:"tier";s:8:"standard";};m:a:0:{}}}""", object_hook=phpserialize.phpobject), {'525f70091c4bd_ja': None})
def unserialize_session(val):
session = {}
groups = re.split(b"([a-zA-Z0-9_]+)\|", val)
if len(groups) > 2:
groups = groups[1:]
groups = zip(*([iter(groups)] * 2))
for key, php_value in groups:
session[key] = phpserialize.loads(php_value)
return session
def decrypt(bstring, key):
#global key
dic = json.loads(base64.b64decode(bstring).decode())
mac = dic['mac']
value = bytes(dic['value'], 'utf-8')
iv = bytes(dic['iv'], 'utf-8')
if mac == hmac.new(key, iv+value, hashlib.sha256).hexdigest():
return loads(mcrypt_decrypt(base64.b64decode(value), base64.b64decode(iv), key)).decode()
return ''
def verify_template():
''' Valid template should be able to be treated as php object stream & converted to python object '''
with open('./message_template.txt', 'rb') as f:
message = f.read().decode()
buf = message.encode()
po = php.loads(buf, object_hook=object_hook, decode_strings=True)
print(json.dumps(po))
def import_from_db(dbname, old=0, cur=get_cur_ts()):
with db.connect(dbname) as cur:
cur.execute(import_query, (ts,cur))
res = cur.fetchall()
urls = []
for x in res:
ser = x[0]
data = phpserialize.loads(ser)
urls.append((data['4::url'], x[1]))
dump(dbname, urls)
def capabilities(self):
capabilities = []
roles_data = phpserialize.loads(
WpOptions.objects.using('wordpress')\
.get(option_name='wp_user_roles').option_value)
for role in self.roles:
role_capabilities = roles_data.get(role).get('capabilities')
for capability, enabled in role_capabilities.iteritems():
if enabled:
capabilities.append(capability)
return set(capabilities)
def read_session_data(self): try: session_id = self.request.cookies[SESSION_NAME] session = DBSession.query(Session).filter_by(session_id=session_id).first() try: data = phpserialize.loads(session.data,array_hook=OrderedDict) return data except Exception, e: return False except Exception, e: return False
def phpserialized(serialized):
""" convert to python string list
"""
from phpserialize import loads
from phpserialize import dict_to_list
try:
serialized = loads(serialized)
serialized = dict_to_list(serialized)
except ValueError:
pass
return serialized
def load_links():
return dict(
map(
lambda path_links: (path_links[0], [link.decode("windows-1251") for link in path_links[1].values()] if isinstance(path_links[1], dict) else path_links[1]),
phpserialize.loads(
urllib2.urlopen(urllib2.Request(
"http://dispenser-01.sape.ru/code.php?user={0}&host={1}".format(config.sape_user_id, config.sape_host)
)).read()
).items()
)
)
def GetAdditionalCollaborationEvents(vars):
# DB connections
# --------------
s = vars['source']
forum_db_selector = db.Selector(s['host'], s['user'], s['password'], s['port'], s['forum_db'])
oe_type_map = moocdb_utils.GetObservedEventTypeMap(vars)
events = []
# Find the id of the first forum_post of each thread. This is the item_id under which the event will be registered
q = "SELECT id, thread_id FROM forum_posts WHERE id IN (SELECT min(id) FROM forum_posts GROUP BY thread_id)"
thread_id_to_first_post_id = {x['thread_id']: x['id'] for x in forum_db_selector.query(q)}
table_name = "kvs_course.{}.forum_readrecord".format(vars['source']['course_id']) if vars['source']['platform_format'] == 'coursera_1' else "kvs_course.forum_readrecord"
q = "SELECT * FROM `{}`".format(table_name)
rows = forum_db_selector.query(q)
for row in rows:
key_parts = row['key'].split(".")
uoid_str = key_parts[1]
try:
int(uoid_str)
except:
continue
user_original_id = int(uoid_str)
if user_original_id not in vars['hash_map']['list_raw']:
continue
forum_id = int(key_parts[0].replace('forum_', ''))
value = phpserialize.loads(row['value'])
if "_all" in value.keys():
events.append({
'user_original_id': user_original_id,
'observed_event_type_id': oe_type_map['forum_visit'],
'item_original_id': forum_id,
'item_type': 'forums',
'observed_event_timestamp': datetime.fromtimestamp(value["_all"]),
'observed_event_data': json.dumps({}),
})
else:
for k in value.keys():
if k in thread_id_to_first_post_id.keys():
events.append({
'user_original_id': user_original_id,
'observed_event_type_id': oe_type_map['forum_post_read'],
'item_original_id': 'post_' + str(thread_id_to_first_post_id[k]),
'item_type': 'forum_posts',
'observed_event_timestamp': datetime.fromtimestamp(value[k]),
'observed_event_data': json.dumps({}),
})
return events
def findPictures(self, sql):
pics = self.touchDb(sql)
for a in pics:
try:
a["meta"] = phpserialize.loads(a["meta_data"].encode('utf-8'),decode_strings=True)
del a["meta_data"]
except:
print "META DATA ERROR:", a["meta_data"]
a["src"] = os.path.join(self.basepath, a["gpath"], a["filename"])
return pics
def load_data(self):
"""
Load cached links and refresh them from sape.ru site if they is too old.
"""
if not os.path.exists(self.db_file):
try:
file(self.db_file, 'w').write('')
os.chmod(self.db_file, 0666)
except IOError:
raise SapeException('Could not create %s' % self.db_file)
mtime = datetime.fromtimestamp(os.stat(self.db_file).st_mtime)
check_time = datetime.now() - timedelta(seconds=self.cache_lifetime)
self.db_file_mtime = mtime
self.db_file_updated = False
if mtime < check_time or not os.path.getsize(self.db_file):
self.db_file_updated = True
new_mtime = check_time + timedelta(seconds=self.cache_reloadtime)
ts = time.mktime(new_mtime.timetuple())
os.utime(self.db_file, (ts, ts))
for server in self.server_list:
data = self.fetch_remote_file(server, self.dispenser_path)
if data.startswith('FATAL ERROR'):
raise SapeException(data)
else:
try:
# check the integrity of data
phpserialize.loads(data)
except:
raise SapeException('Could not deserialize repsonse from server')
else:
self._write(self.db_file, data)
data = self._read(self.db_file)
return phpserialize.loads(data)
